Question

我在下面有一个简单的应用程序。我在Laravel中关闭了查询日志记录，我尽可能不设置，但是这个函数在我2GB Linode上的RAM之前只处理大约800条记录。我知道我问了很多你们，但我似乎无法看到我在哪里泄漏记忆。

实际上只有两个主要步骤。

步骤1 - 将记录从临时表移至生产

class ListingMigrator
{
    public function __construct($tempListing, $feed)
    {
        $this->tempListing = $tempListing;
        $this->listing = $this->listingInstance();
        $this->feed = $feed;
    }

    public static function migrateListing($listing, $feed)
    {
        $instance = new static($listing, $feed);
        return $instance->migrate();
    }

    public function migrate()
    {
        $this->addExternalData();
        $this->populateListing();
        $this->processPhotos();
        $this->deleteTempListing();
    }

    private function listingInstance()
    {
        DB::connection()->disableQueryLog();
        $listing = Listing::findByMud($this->tempListing->matrix_unique_id);
        return $listing ?: new Listing;
    }

    private function processPhotos()
    {
        $retsApi = new RetsFeedApi($this->feed);
        /* Initialize Object */
        $rets = $retsApi->findMostRecent();
        $photos = $rets->getPhotosForListing($this->listing->matrix_unique_id);
                        foreach ($photos as $photo)
                        {
                            $uploader = new PhotoProcessor($this->listing, $photo);
                            $uploader->process();
                        }
    }

    private function populateListing()
    {
        DB::connection()->disableQueryLog();
        $this->listing->fill($this->tempListing->toArray());
        $this->listing->imported_at = $this->tempListing->created_at;
        $this->listing->board = $this->tempListing->board;

        return $this->listing->save();
    }

    private function addExternalData()
    {
        // Get Google lattitude and longitude
        $googlecoords = getGoogleMapInfo($this->tempListing->FullAddress, $this->tempListing->City);
        $this->listing->GoogleLat = $googlecoords['GoogleLat'];
        $this->listing->GoogleLong = $googlecoords['GoogleLong'];

        // Add or update the Subdivision Table (helper function)
        $subdivisiondata = SubdivisionUpdate($this->tempListing->board, $this->tempListing->SubCondoName, $this->tempListing->Development);
        $this->listing->SubdivisionID = $subdivisiondata['id'];
    }

    private function deleteTempListing()
    {
        return $this->tempListing->delete();
    }
}

第2步 - 下载照片并重新上传到Amazon S3

class PhotoProcessor
{
  public function __construct(Listing $listing, $photoData)
  {
    $this->bucket       = 'real-estate-listings';
    $this->s3           = App::make('aws')->get('s3');
    $this->tempFileName = 'app/storage/processing/images/retsphotoupload';
    $this->photoData    = $photoData;
    $this->listing      = $listing;
    $this->photo        = new RetsPhoto;
  }

  public function process()
  {
    $this->storeTempFile();
    $this->storeFileInfo();
    $this->buildPhoto();

    $success = $this->pushToS3();

    // if Result has the full URL or you want to build it, add it to $this->photo
    DB::connection()->disableQueryLog();
    $this->listing->photos()->save($this->photo);  
    $this->removeTempFile();
    unset ($this->photoData);
    return $success;
  }

  private function storeTempFile()
  {
    return File::put($this->tempFileName, $this->photoData['Data']) > 0;
  }

  private function storeFileInfo()
  {
    $fileInfo = getimagesize($this->tempFileName);
    // Could even be its own object
    $this->fileInfo = [
    'width'     => $fileInfo[0],
    'height'    => $fileInfo[1],
    'mimetype'  => $fileInfo['mime'],
    'extension' => $this->getFileExtension($fileInfo['mime'])
    ];
  }

  private function buildPhoto()
  {
    $this->photo->number = $this->photoData['Object-ID']; // Storing this because it is relevant order wise
    $this->photo->width  = $this->fileInfo['width'];
    $this->photo->height = $this->fileInfo['height'];
    $this->photo->path   = $this->getFilePath();
  }

  private function getFilePath()
  {
    $path   = [];
    if ($this->listing->City == NULL)
    {
      $path[] = Str::slug('No City');
    }
    else
    {
      $path[] = Str::slug($this->listing->City, $separator = '-'); 
    }

    if ($this->listing->Development == NULL)
    {
      $path[] = Str::slug('No Development');
    }
    else
    {
      $path[] = Str::slug($this->listing->Development, $separator = '-');  
    }

    if ($this->listing->Subdivision == NULL)
    {
      $pathp[] = Str::slug('No Subdivision');
    }
    else
    {
      $path[] = Str::slug($this->listing->Subdivision, $separator = '-');  
    }

    if ($this->listing->MLSNumber == NULL)
    {
      $pathp[] = Str::slug('No MLSNumber');
    }
    else
    {
      $path[] = Str::slug($this->listing->MLSNumber, $separator = '-');
    }

      $path[] = $this->photoData['Object-ID'].'.'.$this->fileInfo['extension'];

      return strtolower(join('/', $path));
  }

  private function pushToS3()
  {
    return $this->s3->putObject([
      'Bucket'     => $this->bucket,
      'Key'        => $this->photo->path,
      'ContentType'=> $this->fileInfo['mimetype'],
      'SourceFile' => $this->tempFileName
    ]);
  }

  private function getFileExtension($mime)
  {
    // Use better algorithm than this
    $ext = str_replace('image/', '', $mime);
    return $ext == 'jpeg' ? 'jpg' : $ext;
  }

  private function removeTempFile()
  {
    return File::delete($this->tempFileName);
  }
}

编辑以显示RetsPhoto

class RetsPhoto extends Eloquent {

    protected $table = 'rets_property_photos';

    public function listing() {

        return $this->belongsTo('Listing', 'matrix_unique_id', 'matrix_unique_id');
    }

}

编辑＃2：Chunk Call 这是在app /命令中，唯一的东西是下面的fire（）函数：

public function fire()
{
    // Turn off query logging
    DB::connection()->disableQueryLog();

    $feeds = RetsFeed::where('active','=',1)->get();
    foreach ($feeds as $feed)
    {

        $class = "TempListing{$feed->board}";

        $listings = $class::orderBy('MatrixModifiedDT','desc');

        $listings->chunk(50, function($listings) use($feed) {
            $listings->each(function($listing) use ($feed) {
                ListingMigrator::migrateListing($listing,$feed);
                echo "Feed: $feed->board\r\n";
                echo "SubcondoName: $listing->SubCondoName\r\n";
                echo "Development: $listing->Development\r\n";
                echo "\r\n";
            });
        });
    }

}

Answer 1

我想我已经明白了。

您的系统会在内存中保存所有照片数据。正如未设置（$ this-＆gt; photoData）所见;
1. 问题是您需要先完成处理功能。您的应用程序不太可能正在处理任何照片，因此当您继续从文件系统中抓取它们时，在您处理单个照片之前就会耗尽内存。

要确认这一点，只需抓取1个文件而不使用chunk方法。我对Laravel并不熟悉，它可能同时抓住所有文件并吃掉ram。

您可以使用memory_get_usage（true）进行一些跟踪，以找出ram被吃掉的确切位置。我建议先分析火法。

为什么我的应用程序内存不足？使用chunk取消设置变量

1 个答案: