Question

我需要使用Symfony从myqsl数据库中的csv文件（45 Mo）导入大量数据。我导入了League \ Csv \ Reader库我用学说命令了。可以，但是我很慢。我该如何加速呢？

我试图：

添加：$ this-> em-> flush（）之后的$ this-> em-> clear（）;
adding：//禁用SQL日志记录：以避免大量内存丢失。 $this->em->getConnection()->getConfiguration()->setSQLLogger(null);

。

namespace App\Command;

use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use App\Entity\Developer;
use App\Entity\BadgeLabel;
use Doctrine\ORM\EntityManagerInterface;
use League\Csv\Reader;

class CsvImportCommand extends Command
{
    public function __construct(EntityManagerInterface $em){

        parent::__construct();

        $this->em = $em;
    }

    // the name of the command (the part after "bin/console")
    protected static $defaultName = 'app:import-developpers';

    protected function configure()
    {
        $this
        // the short description shown while running "php bin/console list"
        ->setDescription('Import a new developper.')

        // the full command description shown when running the command with
        // the "--help" option
        ->setHelp('This command allows you to import a develpper...')
    ;

    }

    protected function execute(InputInterface $input, OutputInterface $output)
    {
        $io = new SymfonyStyle($input, $output);
        $io->title('Importation en cours');


        $reader = Reader::createFromPath('%kernel.root_dir%/../src/Data/developers_big.csv')
            ->setHeaderOffset(0)
        ;
        $results = $reader->getrecords();
        $io->progressStart(iterator_count($results));

        //Disable SQL Logging: to avoid huge memory loss.
        $this->em->getConnection()->getConfiguration()->setSQLLogger(null);


        foreach ($results as $row) {
            $developer = $this->em->getRepository(Developer::class)
            ->findOneBy([
                'firstName' => ($row['FIRSTNAME']),
                'lastName'=> ($row['LASTNAME'])
            ])
            ;

            if (null === $developer) {
                $developer = new developer;
                $developer
                    ->setFirstName($row['FIRSTNAME'])
                    ->setLastName($row['LASTNAME']);
                $this->em->persist($developer);
                $this->em->flush();
                $this->em->clear();             
            }

            $badgeLabel = $this->em->getRepository(BadgeLabel::class)
                ->findOneBy([
                    'name' => ($row['BADGE LABEL']),
                    'level'=> ($row['BADGE LEVEL'])
                ])
            ;

            if (null === $badgeLabel) {
                $badgeLabel = new BadgeLabel;
                $badgeLabel
                    ->setName($row['BADGE LABEL'])
                    ->setLevel($row['BADGE LEVEL']);
                $this->em->persist($badgeLabel);
                $this->em->flush();
                $this->em->clear();

            }
            $developer
                ->addBadgeLabel($badgeLabel);

            $io->progressAdvance();
        }

        $this->em->flush();
        $this->em->clear();

        $io->progressFinish();
        $io->success('Importation terminée avec succès');
    }
}

该命令的运行速度变慢。 15分钟后，Mysql数据库中只有32％的文件被上传。我希望最多2分钟之内

Answer 1

方法1 ：（不是最好的）

调用flush方法时，Symfony遍历所有侦听器。因此，您可以避免刷新每个循环。您可以使用以下代码替换每次冲洗：

if (0 === ($batchSize++ % $input->getOption('fetch'))) {
    $this->entityManager->flush();
    $this->entityManager->clear();
}

可以在configure方法中声明

fetch选项：

    const BATCH_SIZE = 1000; // As example

    /**
     * Configure the command.
     */
    protected function configure()
    {
        $this
        // the short description shown while running "php bin/console list"
        ->setDescription('Import a new developper.')

        //This option helps you to find a good value and use BATCH_SIZE constant as default
        ->addOption('fetch', 'f', InputArgument::OPTIONAL, 'Number of loop between each flush', self::BATCH_SIZE)

        // the full command description shown when running the command with
        // the "--help" option
        ->setHelp('This command allows you to import a develpper...')
    ;

方法2：更高效

您可以创建一个命令，该命令将所有带有更新的SQL查询写入或插入到sql文件中。然后，启动一个本机命令来读取文件并执行查询。

方法3：使用DBAL 如评论中所建议，您可以使用DBAL来避免使用Doctrine进行不必要的对象水化。

使用symfony命令导入大量的csv数据，使用原理太慢

1 个答案: