如何有效地将一百万个对象写入CSV文件？

Question

我正在尝试编写CSV文件，该数据库中有数百万条记录。我在数据库中总共有800万条记录，每条记录都有5个属性。因此，CSV文件将具有800万行和5列。我需要以最有效的方式执行此操作，但是我不能只花4分钟就写一百万个字。我试图增加缓冲的作家的大小，我也尝试直接使用文件作家，但我有4分钟。我的代码有什么改进吗？

我的代码：

@Component
public class SaveCSVAdminImsiHelper {

    @Autowired
    private Environment environment;

    @Autowired
    private WmAdminImsisResourceHelper wmAdminImsisResourceHelper;

    @Autowired
    private WmPushedImsiService wmPushedImsiService;

    @PersistenceContext
    private EntityManager entityManager;

    private List<String> imsisCsvColumns;

    private Logger imsiLogger = LoggerFactory.getImsiLogger(this.getClass().getName());

    private void initCsvColumns() {
        imsisCsvColumns = new ArrayList<>();
        imsisCsvColumns.add(environment.getProperty("CSV_IMSINUMBER"));
        imsisCsvColumns.add(environment.getProperty("CSV_USERNAME"));
        imsisCsvColumns.add(environment.getProperty("CSV_STARTDATE"));
        imsisCsvColumns.add(environment.getProperty("CSV_EXPIREDATE"));
        imsisCsvColumns.add(environment.getProperty("CSV_DIRTY"));
    }

    private void formatAndWriteField(BufferedWriter writer, String field, char separator, boolean isLastField) throws IOException {
        field = field.replace("\"", "\"\"");

        if (field.indexOf(separator) > -1 || field.indexOf('"') > -1) {
            field = '"' + field + '"';
        }

        writer.append(field);
        if (!isLastField) {
            writer.append(separator);
        }
    }

    private void writeColumnsTitle(BufferedWriter writer, List<String> columnsTitle, char separator) throws IOException {
        for (Iterator<String> keysIterator = columnsTitle.iterator(); keysIterator.hasNext();) {
            String column = keysIterator.next();

            if (keysIterator.hasNext()) {
                formatAndWriteField(writer, column, separator, false);
            } else {
                formatAndWriteField(writer, column, separator, true);
            }
        }
        writer.newLine();
    }

    private String checkForImsiDirty(WmPushedImsi wmPushedImsi) {
        return (wmPushedImsi.getFailedPushBatch() != null || wmPushedImsi.getFailedDeleteBatch() != null) ? "Yes" : "No";
    }


    private void writeImsiProperties(BufferedWriter writer, List<WmPushedImsi> wmPushedImsis, char separator) throws Exception {
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
        for (WmPushedImsi wmPushedImsi : wmPushedImsis) {
            formatAndWriteField(writer, wmPushedImsi.getImsiNumber(), separator, false);
            formatAndWriteField(writer, wmPushedImsi.getUsername(), separator, false);
            formatAndWriteField(writer, simpleDateFormat.format(wmPushedImsi.getStartDate()), separator, false);
            formatAndWriteField(writer, simpleDateFormat.format(wmPushedImsi.getExpireDate()), separator, false);
            formatAndWriteField(writer, checkForImsiDirty(wmPushedImsi), separator, true);
            writer.newLine();
        }
    }

    private File writeFinalCsv(String fileName, char separator, WmAdminImsisResource wmAdminImsisResource) throws Exception {
        File file = new File(fileName);
        int PAGE_SIZE = 300_000;
        Map<String, String> filterMap = wmAdminImsisResourceHelper.createFilteringProperties(wmAdminImsisResource);
        Map<String, String> sortingMap = wmAdminImsisResourceHelper.createSortingProperties(wmAdminImsisResource);

        long pages = wmPushedImsiService.countAllWithSortCriteriaAndFilter(filterMap) / PAGE_SIZE;
        initCsvColumns();
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(file), 31457280)) {

            writeColumnsTitle(writer, imsisCsvColumns, separator);

            for (int page = 0; page <= pages; page++) {
                List<WmPushedImsi> wmPushedImsis = wmPushedImsiService.findAllWithSortCriteriaAndFilter(filterMap, sortingMap, page, PAGE_SIZE);
                writeImsiProperties(writer, wmPushedImsis, separator);
                entityManager.getEntityManagerFactory().getCache().evictAll();
            }
        } catch (IOException e) {
            imsiLogger.log(Level.SEVERE, environment.getProperty("wlm.error.failedToSaveCsv"), e);
        }

        return file;
    }

    private void createImsiZip(WmAdminImsisResource wmAdminImsisResource, File imsiCsvZip, String fileNameWithoutExtension) throws Exception {
        try (FileOutputStream fileOutputStream = new FileOutputStream(imsiCsvZip.getCanonicalFile());
                ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream)) {
            imsiCsvZip.deleteOnExit();

            String fileName = fileNameWithoutExtension + environment.getProperty("CSVEXTENSION");
            File imsisCsv = writeFinalCsv(fileName, ',', wmAdminImsisResource);
            String fileNameInZip = environment.getProperty("CSVNAME") + environment.getProperty("CSVEXTENSION");
            WsUtils.addFileToZipWithOtherName("", imsisCsv, zipOutputStream, fileNameInZip);
            if (imsisCsv.exists()) {
                imsisCsv.delete();
            }

        } catch (IOException e) {
            imsiLogger.log(Level.SEVERE, environment.getProperty("wlm.error.failedToSaveCsv"), e);
        }
    }

    public String downloadImsiCsvZip(WmAdminImsisResource wmAdminImsisResource, StringBuilder stringBuilder) {
        try {
            String path = "";
            String html = environment.getProperty("HTML") + File.separator;
            String htmlCsv = html + environment.getProperty("CSV") + File.separator;
            SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss.SSS");
            String startPath = environment.getProperty("STARTPATH") + File.separator;
            File csvDirectory = new File(htmlCsv);
            if (!csvDirectory.exists()) {
                csvDirectory.mkdir();
            }
            csvDirectory.deleteOnExit();
            String fileNameWithoutExtension = htmlCsv + environment.getProperty("CSVNAME") + simpleDateFormat.format(new Date());
            File imsisCsvZip = new File(fileNameWithoutExtension + environment.getProperty("ZIPEXTENSION"));
            createImsiZip(wmAdminImsisResource, imsisCsvZip, fileNameWithoutExtension);
            if (imsisCsvZip.exists()) {
                path = imsisCsvZip.getPath();
            }
            if (path.startsWith(html)) {
                path = path.substring(html.length());
            }
            path = path.replaceAll("\\\\", "/");

            return startPath + path;
        } catch (Exception e) {
            imsiLogger.log(Level.SEVERE, environment.getProperty("wlm.error.failedToSaveCsv"), e);
            stringBuilder.append(e);
        }
        return "";
    }



}