Question

如何解决问题

warning: conflicting chdir during another chdir block

我获取了城市中的所有位置，并创建了一个附加文件的文件夹如何正确优化代码并实现正确的工作？如何检查文件夹是否存在，然后将新文本添加到现有文件中？

require 'open-uri'
require 'JSON'
require 'thread'

def scrape_instagram_city_page(page)
    cityArray = []
    id = 0
    begin
        instagram_source = open(page).read
        content = JSON.parse(instagram_source.split("window._sharedData = ")[1].split(";</script>")[0])
        locationName = content['entry_data']['LocationsDirectoryPage'][0]['city_info']['name']
        nextpage = content['entry_data']['LocationsDirectoryPage'][0]['next_page'] 
        Dir.mkdir("#{locationName}")
        loop do
            id +=1
            instagram_source = open(page+"?page=#{id}").read
            content = JSON.parse(instagram_source.split("window._sharedData = ")[1].split(";</script>")[0])
            locationsList = content['entry_data']['LocationsDirectoryPage'][0]['location_list']
            locationsList.each do |hash|
                cityArray.push(hash['id'].to_i)
            end
            if nextpage == "null"
                break
            end
        Dir.chdir("#{locationName}") do
            fileName = "#{locationName}.txt"
            File.open(fileName, 'w') do |file|
                cityArray.each do |item|
                    file << "https://www.instagram.com/explore/locations/#{item}/\n"
                end
            end
        end
        end
    rescue Exception => e
        return nil
    end
end

threads = []
city = ["https://www.instagram.com/explore/locations/c2269433/dhewng-thailand/","https://www.instagram.com/explore/locations/c2260532/ban-poek-thailand/","https://www.instagram.com/explore/locations/c2267999/ban-wang-takrai-thailand/","https://www.instagram.com/explore/locations/c2255595/ban-nong-kho-thailand/","https://www.instagram.com/explore/locations/c2252832/ban-na-khum-thailand/","https://www.instagram.com/explore/locations/c2267577/ban-wang-khaen-thailand/","https://www.instagram.com/explore/locations/c2248064/ban-khung-mae-luk-on-thailand/","https://www.instagram.com/explore/locations/c2243370/ban-hua-dong-kheng-thailand/","https://www.instagram.com/explore/locations/c2269271/chieng-sean-thailand/","https://www.instagram.com/explore/locations/c2256442/ban-nong-phiman-thailand/","https://www.instagram.com/explore/locations/c2246490/ban-khlong-khwang-thai-thailand/"]
city.each do |page|
    threads << Thread.new do
        scrape_instagram_city_page "#{page}"
    end
end

threads.each(&:join)

Answer 1

在回答问题之前，我会注意到抓取网站往往违反该网站的服务条款。你应该检查一下，确保你没有做违法的事。

chdir更改的“当前目录”是所有线程共享的进程范围设置。这就是为什么当两个线程同时尝试更改它时会出现异常的原因。它与您创建的线程数无关。

要避免此问题，请不要更改当前目录。只需在路径中包含目录：

def scrape_instagram_city_page(page)
    cityArray = []
    id = 0
    begin
        instagram_source = open(page).read
        content = JSON.parse(instagram_source.split("window._sharedData = ")[1].split(";</script>")[0])
        locationName = content['entry_data']['LocationsDirectoryPage'][0]['city_info']['name']
        nextpage = content['entry_data']['LocationsDirectoryPage'][0]['next_page'] 
        Dir.mkdir("#{locationName}")
        loop do
            id +=1
            instagram_source = open(page+"?page=#{id}").read
            content = JSON.parse(instagram_source.split("window._sharedData = ")[1].split(";</script>")[0])
            locationsList = content['entry_data']['LocationsDirectoryPage'][0]['location_list']
            locationsList.each do |hash|
                cityArray.push(hash['id'].to_i)
            end
            if nextpage == "null"
                break
            end
            fileName = "#{locationName}/#{locationName}.txt"
            File.open(fileName, 'w') do |file|
                cityArray.each do |item|
                    file << "https://www.instagram.com/explore/locations/#{item}/\n"
                end
            end
        end
    rescue Exception => e
        return nil
    end
end

如果城市数量非常大，如何正确使用线程？

1 个答案: