Question

我在rails上有这个rake任务填充我的数据库，它需要每天运行。

require 'open-uri'
require 'csv'
namespace :tm do
  task reload: :environment do
    gzipped = open('csv link')
    csv_text = Zlib::GzipReader.new(gzipped).read
    csv = CSV.parse(csv_text, headers: true)

    csv.each do |row|
      if row[4] == 'logo url'
      else
      tmdate = Date.parse(row[10]).strftime('%Y-%m-%d')
      viatmdate = Date.parse(row[10]).strftime('%d/%m/%Y')
      swtmdate = row[10]
      tmlocation = row[6].split('at ')[1]
      place = row[11].split('|')[1]
      place1 = row[11].split('|')[2]
      place2 = row[11].split('|')[3]
      location = '' + place + ', ' + place1 + ', ' + place2 + ''
      tmtime = row[9]
      text = row[7].gsub('text', '')
      if text.include? '&#xFFFD;&#xFFFD'
        eventname = text.gsub('&#xFFFD;&#xFFFD', 'e')
      else
        eventname = text.gsub(/[ªÀÈÌÒÙàèìòùÁÉÍÓÚáéíóúÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿ]/, '')
      end

      if text.include? '&#xFFFD;&#xFFFD'
        tmname = text.gsub('&#xFFFD;&#xFFFD', 'e')
      else
        tmname = text.gsub(/[ªÀÈÌÒÙàèìòùÁÉÍÓÚáéíóúÂÊÎÔÛâêîôûÃÑÕãñõÄËÏÖÜŸäëïöüÿ]/, '')
    end
    if text.include? ' -'
      tmnamesplit = text.split(' -')[0]
    end
    if tmname[/[^0-9]/].present?
      tmnamenn = tmname.gsub(/[^0-9]/i, '')
    end
        text2urldb = text2.where('eventtitle ILIKE ? AND eventdoortime = ? ', "%#{tmname.gsub(/[\-\:\ ]/, '%')}%", tmdate.to_s).first
        text3urldb = text3.where('product_name ILIKE ? AND delivery_time = ? AND valid_from = ?', "%#{tmname}%", tmtime.to_s, tmdate.to_s).first

        text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{place}%").first

        if tmnamesplit.present?
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ?', "%#{tmnamesplit}%", viatmdate.to_s).first
          end
          if text3urldb.blank?
            text3urldb = text3.where('product_name ILIKE ? AND delivery_time = ? AND valid_from = ?', "%#{tmnamesplit}%", tmtime.to_s, tmdate.to_s).first
          end
        end

        if text1urldb.blank?
          text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{location}%").first
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ?', "%#{tmname}%", viatmdate.to_s).first
          end
          if text1urldb.blank?
            text1urldb = text1.where('product_name ILIKE ? AND specifications = ? AND promotional_text = ?', "%#{tmname}%", viatmdate.to_s, "%#{tmlocation}%").first
          end
        end



      if text1urldb.present?
        vurl = text1urldb.merchant_deep_link
        txt = vurl
        re1 = '.*?' # Non-greedy match on filler
        re2 = '(?:[a-z][a-z]+)' # Uninteresting: word
        re3 = '.*?' # Non-greedy match on filler
        re4 = '(?:[a-z][a-z]+)' # Uninteresting: word
        re5 = '.*?' # Non-greedy match on filler
        re6 = '(?:[a-z][a-z]+)' # Uninteresting: word
        re7 = '.*?' # Non-greedy match on filler
        re8 = '(?:[a-z][a-z]+)' # Uninteresting: word
        re9 = '.*?' # Non-greedy match on filler
        re10 = '(?:[a-z][a-z]+)'    # Uninteresting: word
        re11 = '.*?'    # Non-greedy match on filler
        re12 = '((?:[a-z][a-z]+))'  # Word 1
        re = (re1 + re2 + re3 + re4 + re5 + re6 + re7 + re8 + re9 + re10 + re11 + re12)
        m = Regexp.new(re, Regexp::IGNORECASE)
        if m.match(txt)
          word1 = m.match(txt)[1]
        end
      end


      gmiurl = text3urldb.merchant_deep_link if text3urldb.present?

      gigurl = text2urldb.eventurl if text2urldb.present?

      api = HTTParty.get(URI.encode('text url' + tmname + '&when_from=' + swtmdate)).parsed_response
      api1 = api['Paging']
      api2 = api1['TotalResultCount']

      if api1.blank?
        newapi = HTTParty.get(URI.encode('texturl' + tmnamenn + '&when_from=' + swtmdate)).parsed_response
        paging = newapi['Paging']
        api2 = paging['TotalResultCount']
        if newapi.blank?
          apisplit = HTTParty.get(URI.encode('texturl' + tmnamesplit + '&when_from=' + swtmdate)).parsed_response
          pagingsplit = apisplit['Paging']
          api2 = pagingsplit['TotalResultCount']
        end
      end
      text1 = vurl
      text3 = gmiurl
      text2 = gigurl

      if api2 == 0
      else
        swurl = api['Events'].first['SwURL']
      end
      event = Event.find_by(time: row[9], date: row[10], eventname: eventname, eventvenuename: location)

      if event.present?
        event.update(event_type: word1, text: row[8], eventimage: row[4], textlink: swurl, text1link: text1, text3url: text3, text2url: text2)
      else
        Event.create(time: row[9], date: row[10], event_type: word1, text: row[8], eventimage: row[4], eventname: eventname, eventvenuename: location, textlink: swurl, text1link: text1, text3url: text3, text2url: text2)
      end

    end
    end
  end
end

现在我愿意做任何事情，csv链接是一个到csv的api链接。我愿意将其分解为多个文件，问题我不知道运行需要多长时间，所以它必须是一次完整的下一次rake任务。

目前，这需要大约2-3个小时才能完成并填充数据库。

任何想法如何加快速度？

由于

萨姆

Answer 1

你可以：
- 将后台的作业用于此类的周期性任务。（例如：sidekiq）
- 确保正确设置了DB索引。

要对此问题获得更多回复：
- 为变量使用rails命名约定
- 清理你的代码，一些任务使其无法阅读 - 使用方法做一些冗余的东西

Answer 2

添加数据库索引。您在那里进行了大量查询，并且productname，specifications，eventtitle等列都应被视为具有索引。

加快2-3小时的佣金任务

2 个答案: