Rails CSV导入重复创建

时间:2013-10-15 20:39:58

标签: ruby-on-rails ruby csv import

我正在将大量数据加载到我的应用程序中。我有超过40个模型,我正在加载它的数据。除了其中一个表之外的所有表都进口正常。在一个表中我遇到问题,每一行都是重复的,所以不是有234,225行,而是最终得到468,450行。绝对不是我想要的。每一行都完全重复。

作为此导入的一部分,我也将数据结构扁平化。所以我将查找字段填充到主sku表中。

这是我用来导入这个skus表的代码。

命名空间:upload_data do

  desc "Load Skus"
  task :load_skus => :environment do
    require 'csv'
    require 'net/ftp'
    start_time = Time.now
    ftp = Net::FTP.new('<ftp location>')
    ftp.login(user = "<username>", passwd = "<password>")
    ftp.passive = true
    ftp.get("sku.csv", File.basename( "/data/sku.csv"))
    ftp.quit()
    upload = File.join('sku.csv')
      logcount=0
      CSV.foreach(upload, :headers => false) do |row|
        # roa = Roa.find_by_roa_code(row[21])
        # if roa
        #   roa_name = roa.roa_name
        # else
        #   roa_name = nil
        # end  

        #ROA      
        roa = Roa.find_by_roa_code(row[21])
        roa_name = roa && roa.roa_name

        #THER   
        ther = Ther.find_by_thera_class_code_int(row[50] && row[50].to_i)
        thera_class_desc = ther && ther.thera_class_desc

        #DEA
        dea = Dea.find_by_dea_class_code(row[45])
        dea_class_desc = dea && dea.dea_class_desc

        #LBLY2K
        lbly2k = Lbly2k.find_by_mfg_code(row[56])
        labeler_code = lbly2k && lbly2k.labeler_code
        rebate_status = lbly2k && lbly2k.rebate_status
        rebate_effective_date  = lbly2k && lbly2k.rebate_effective_date   
        rebate_termination_date = lbly2k && lbly2k.rebate_termination_date   

        #FORM      
        form = Form.find_by_master_form_code(row[22])
        form_desc = form && form.form_desc

        #PRODCAT      
        prodcat = Prodcat.find_by_product_category(row[26] && row[26].to_i.to_s)
        product_cat_desc = prodcat && prodcat.product_cat_desc

        #EXPGENRC      
        expgenrc = Expgenrc.find_by_gcr_code(row[52])
        gcr_exp_name = expgenrc && expgenrc.gcr_exp_name

        #GCR      
        gcr = Gcr.find_by_gcr_code(row[52])
        gcr_name = gcr && gcr.gcr_name

        #MFORM      
        mform = Mform.find_by_master_form_code(row[22])
        master_form_desc = mform && mform.master_form_desc

        #Orange Book Data
        orange = Orange.find_by_orange_book_code(row[31])
        orange_book_desc = orange && orange.orange_book_desc

        #EXDRUG      
        exdrug = Exdrug.find_by_exceptional_drug(row[44])
        exceptional_drug_desc = exdrug && exdrug.exceptional_drug_desc

        #MEASURE      
        measure = Measure.find_by_measure_code(row[39])
        measure_desc = measure && measure.measure_desc

        #JCODE      
        jcode = Jcode.find_by_ndc(row[13] && row[13].gsub('-', ''))
        puts "found jcode: #{jcode.inspect}"
        j_code = jcode && jcode.j_code
        j_code_desc = jcode && jcode.j_code_desc

        #GFC
        gfc = Gfc.find_by_gcr_code(row[52])
        gfc_master_code = gfc && gfc.gfc_master_code
        gfc_deact = gfc && gfc.gfc_deact 

        #packstren      
        packstren = Packstren.find_by_ndc(row[13] && row[13].gsub('-', ''))
        puts "Found Packstren #{packstren.inspect}"
        package_ID = packstren && packstren.package_ID
        puts "Package_ID is #{package_ID}"
        strength_code = packstren && packstren.strength_code 

        #STRENGTH 
        strength = Strength.find_by_strength_code(strength_code)
        puts "Found strength: #{strength.inspect}"
        strength_name_long = strength && strength.strength_name_long
        puts "Strength name long is #{strength_name_long}"

        #EXPSTREN 
        expstren = Expstren.find_by_strength_code(strength_code)
        strength_exp_name = expstren && expstren.strength_exp_name
        puts "Strength_exp_name is #{strength_exp_name}"

        #AWP
        awp = Awp.find_by_ndc(row[13] && row[13].gsub('-', ''))
        puts "Found awp: #{awp.inspect}"
        effective_date = awp && awp.effective_date
        puts "found effective_date: #{effective_date}"
        manufacturer_cmsid = awp && awp.manufacturer_cmsid
        puts "found manufacturer_cmsid: #{manufacturer_cmsid}"
        package_cmsid = awp && awp.package_cmsid 
        puts "found package_cmsid: #{package_cmsid}"

        Sku.create(:record_change           => row[0], 
                   :record_change_date      => row[1], 
                   :price_change_date       => row[2], 
                   :add_flag                => row[3], 
                   :add_date                => row[4], 
                   :deactivate_flag         => row[5], 
                   :deactivate_date         => row[6], 
                   :reactivate_flag         => row[7], 
                   :reactivate_date         => row[8], 
                   :filler1                 => row[9], 
                   :prev_ndc_awp_eff_date   => row[10], 
                   :prev_ndc_awp_pack_price => row[11], 
                   :prev_ndc_disc_date      => row[12], 
                   :ndc                     => row[13].gsub('-',''),
                   :j_code                  => j_code,
                   :j_code_desc             => j_code_desc,
                   :package_ID              => package_ID,
                   :strength_code           => strength_code,
                   :filler2                 => row[14], 
                   :ndc_orig_config         => row[15].gsub('-',''), 
                   :ndc10                   => row[16].gsub('-',''), 
                   :ndc_change_flag         => row[17], 
                   :prev_ndc                => row[18], 
                   :prev_ndc_config         => row[19], 
                   :prev_ndc10              => row[20], 
                   :roa_code                => row[21], 
                   :roa_name                => roa_name,
                   # :roa_name              => Roa.find_by_roa_code(row[21]) && Roa.find_by_roa_code(row[21]).roa_name,
                   :master_form_code        => row[22], 
                   :master_form_desc        => master_form_desc,
                   :form_desc               => form_desc,
                   :product_name            => row[23], 
                   :additional_desc         => row[24], 
                   :mfg_name_long           => row[25], 
                   :product_category        => row[26], 
                   :product_cat_desc        => product_cat_desc,
                   :solid_liquid            => row[27], 
                   :form_code               => row[28], 
                   :metric_size             => row[29], 
                   :strength_name           => row[30], 
                   :orange_book_code        => row[31],
                   :orange_book_desc        => orange_book_desc, 
                   :filler3                 => row[32], 
                   :orange_book_std_flag    => row[33], 
                   :unit_dose_flag          => row[34], 
                   :dispensing_unit_flag    => row[35], 
                   :package_size            => row[36], 
                   :package_qty_code        => row[37], 
                   :product_size            => row[38], 
                   :measure_code            => row[39],
                   :measure_desc            => measure_desc, 
                   :filler4                 => row[40], 
                   :top_volume_rank         => row[41], 
                   :single_source_flag      => row[42], 
                   :maintenance_drug_flag   => row[43], 
                   :exceptional_drug        => row[44],
                   :exceptional_drug_desc   => exceptional_drug_desc, 
                   :dea_class_code          => row[45],
                   :dea_class_desc          => dea_class_desc, 
                   :filler5                 => row[46], 
                   :filler6                 => row[47], 
                   :desi_drug_flag          => row[48], 
                   :desi_effective_date     => row[49], 
                   :thera_class_code        => row[50],
                   :thera_class_desc        => thera_class_desc, 
                   :generic_class_code      => row[51], 
                   :gcr_code                => row[52],
                   :gcr_exp_name            => gcr_exp_name,
                   :gcr_name                => gcr_name,
                   :gfc_master_code         => gfc_master_code,
                   :gfc_deact               => gfc_deact,
                   :gfc_code                => row[53], 
                   :stc_package_size        => row[54], 
                   :unit_of_measure         => row[55], 
                   :mfg_code                => row[56], 
                   :labeler_code            => labeler_code,
                   :rebate_status           => rebate_status,
                   :rebate_effective_date   => rebate_effective_date,
                   :rebate_termination_date => rebate_termination_date,
                   :wac_start_flag          => row[57], 
                   :wac_p_price             => row[58], 
                   :wac_u_price             => row[59], 
                   :wac_eff_date            => row[60], 
                   :wac1pre_p_price         => row[61], 
                   :wac1pre_u_price         => row[62], 
                   :wac1pre_eff_date        => row[63], 
                   :wac2pre_p_price         => row[64], 
                   :wac2pre_u_price         => row[65], 
                   :wac2pre_eff_date        => row[66], 
                   :filler7                 => row[67], 
                   :filler8                 => row[68], 
                   :filler9                 => row[69], 
                   :awp_start_flag          => row[70], 
                   :awp_p_price             => row[71], 
                   :awp_u_price             => row[72], 
                   :awp_eff_date            => row[73], 
                   :awp1pre_p_price         => row[74], 
                   :awp1pre_u_price         => row[75], 
                   :awp1pre_eff_date        => row[76], 
                   :awp2pre_p_price         => row[77], 
                   :awp2pre_u_price         => row[78], 
                   :awp2pre_eff_date        => row[79], 
                   :dp_start_flag           => row[80], 
                   :dp_p_price              => row[81], 
                   :dp_u_price              => row[82], 
                   :dp_eff_date             => row[83], 
                   :dp1pre_p_price          => row[84], 
                   :dp1pre_u_price          => row[85], 
                   :dp1pre_eff_date         => row[86], 
                   :dp2pre_p_price          => row[87], 
                   :dp2pre_u_price          => row[88], 
                   :dp2pre_eff_date         => row[89], 
                   :ful_start_flag          => row[90], 
                   :ful_p_price             => row[91], 
                   :ful_u_price             => row[92], 
                   :ful_eff_date            => row[93], 
                   :ful_pre_p_price         => row[94], 
                   :ful_pre_u_price         => row[95], 
                   :ful_pre_eff_date        => row[96], 
                   :ful2_pre_p_price        => row[97], 
                   :ful2_pre_u_price        => row[98], 
                   :ful2_pre_eff_date       => row[99], 
                   :srp_start_flag          => row[100], 
                   :srp_p_price             => row[102], 
                   :srp_u_price             => row[102], 
                   :srp_eff_date            => row[103], 
                   :srp1pre_p_price         => row[104], 
                   :srp1pre_u_price         => row[105], 
                   :srp1pre_eff_date        => row[106], 
                   :srp2pre_p_price         => row[107], 
                   :srp2pre_u_price         => row[108], 
                   :srp2pre_eff_date        => row[109])

      logcount += 1
    end
    total_time = Time.now - start_time
      puts "Successfully added #{logcount} Skus. Elapsed time #{total_time} seconds."

  end

end

我正在做什么导致每行导入?

由于

0 个答案:

没有答案