Question

我有一个类似于下面的ETL，除了我们从mysql中提取8个字段并且有8个顶点类和7个边缘类。

{
      "config": {
        "parallel": true
      },
      "extractor": {
        "jdbc": {
          "driver": "com.mysql.jdbc.Driver",
          "url": "jdbc:mysql://<some IP>/car_database",
          "userName": "admin",
          "userPassword": "admin",
          "query": "select uid, type, color, age from car_info;",
          "fetchSize": -2147483648
        }
      },
      "transformers": [
        {
          "vertex": {
            "class": "Car"
          }
        },
        {
          "field": {
            "fieldName": "value",
            "expression": "uid"
          }
        },
        {
          "edge": {
            "class": "HasType",
            "lookup": "Type.value",
            "joinFieldName": "type",
            "unresolvedLinkAction": "CREATE"
          }
        },
        {
          "edge": {
            "class": "HasColor",
            "lookup": "Color.value",
            "joinFieldName": "color",
            "unresolvedLinkAction": "CREATE"
          }
        },
        {
          "edge": {
            "class": "HasAge",
            "lookup": "Age.value",
            "joinFieldName": "age",
            "unresolvedLinkAction": "CREATE"
          }
        },
        {
          "field": {
            "fieldNames" : [
              "uid",
              "type",
              "color",
              "age"
            ],
            "operation": "remove"
          }
        }
      ],
      "loader": {
        "orientdb": {
          "dbURL": "remote:<some IP >/CarGraph",
          "dbType": "graph",
          "txUseLog": false,
          "wal": false,
          "tx": false,
          "batchCommit": 1000,
          "classes": [
            {
              "name": "Car",
              "extends": "V"
            },
             {
                "name": "Type",
                "extends": "V"
              },
              {
                "name": "HasType",
                "extends": "E"
              },
              {
                "name": "Color",
                "extends": "V"
              },
              {
                "name": "HasColor",
                "extends": "E"
              },
              {
                "name": "Age",
                "extends": "V"
              },
              {
                "name": "HasAge",
                "extends": "E"
              }
          ],
          "indexes": [
            {
              "class": "Car",
              "fields": [
                "value:string"
              ],
              "type": "UNIQUE"
            },
            {
              "class": "Type",
              "fields": [
                "value:string"
              ],
              "type": "UNIQUE"
            },
            {
              "class": "Color",
              "fields": [
                "value:string"
              ],
              "type": "UNIQUE"
            },
            {
              "class": "Age",
              "fields": [
                "value:string"
              ],
              "type": "UNIQUE"
            }
          ]
        }
      }
    }

这是我的default-distributed-db-cofig文件：

{
  "autoDeploy": true,
  "hotAlignment": false,
  "executionMode": "undefined",
  "readQuorum": 1,
  "writeQuorum": 1,
  "failureAvailableNodesLessQuorum": false,
  "readYourWrites": true,
  "servers": {
    "*": "master"
  },
  "clusters": {
    "internal": {
    },
    "index": {
    },
    "*": {
      "servers": ["<NEW_NODE>"]
    }
  }
}

我正在尝试实现一个Orient集群，目前在不同的服务器上有2个节点。当我在其中一个图形框上运行导入时（我们当前的QA环境只有200行，prod将是3500万加），我看到加载时间为1-3个节点，即使仲裁设置为1。尝试了5-6种不同的组合，切换并行，仲裁值，不同的服务器设置，但只要加载器指向远程数据库，它就会非常慢。

同样的etl在我的本地盒子上以1500-2000 / s的速度运行到一个图表中。

我发现运行导入的最快方法是将加载程序指向一个图表。然后，一旦完成重新启动分布式服务器并允许2个节点同步。

我有什么遗漏，或者某些服务器配置可能会导致导入速度太慢？东方版2.1.6。

orientdb慢分布式etl导入

0 个答案: