在API中将34000条记录从API插入到sqlite中

时间：2012-03-15 11:34:32

标签： iphone json performance sqlite insert

我必须从从JSON API获取的数据创建一个sqlite数据库。代码工作正常并通过for循环逐个添加它们，但api响应时间为每次命中1秒，因此34000秒加上通过代码在sqlite中插入它们大约需要9个小时。有没有办法加快这个？

编辑：我正在使用Objective C / sqlite3 framework / Xcode 4.2

继承守则...... 

 dbPath=[self.databasePath UTF8String];
if(sqlite3_open(dbPath,&database)==SQLITE_OK)
{
  //   sqlite3_exec(database, "BEGIN", 0, 0, 0);
    const char *sqlstatement="insert into artist values(?,?,?,?,?)";
    sqlite3_stmt *compiledstatement;

    if(sqlite3_prepare_v2(database,sqlstatement , -1, &compiledstatement, NULL)==SQLITE_OK)
    {    
for(i=4611;i<=34803;i++)
{  
    NSURLResponse *response;
    NSError *err;
  NSData *data= [NSURLConnection sendSynchronousRequest:[NSURLRequest requestWithURL:[NSURL URLWithString:[NSString stringWithFormat:@"API&id=%i",i]]] returningResponse:&response error:&err];
    if(data.length>0)
    {
        NSError *err;
        NSDictionary *jsonDict=[NSJSONSerialization JSONObjectWithData:data options:NSJSONReadingAllowFragments error:&err];
        // sqlite3_exec(database, "BEGIN", 0, 0, 0);




                sqlite3_bind_text(compiledstatement,1,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"id"] UTF8String], -1, SQLITE_TRANSIENT);
                sqlite3_bind_text(compiledstatement,2,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"name"] UTF8String], -1, SQLITE_TRANSIENT);
                if([[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"])
                    sqlite3_bind_text(compiledstatement,3,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,3,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"])
                    sqlite3_bind_text(compiledstatement,4,[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,4,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0]objectForKey:@"url"])
                    sqlite3_bind_text(compiledstatement,5,[[[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0] objectForKey:@"url"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,5,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if(sqlite3_step(compiledstatement)==SQLITE_DONE)
                {
                    NSLog(@"done %i",i);
                }
                else NSLog(@"ERROR");


        }
        sqlite3_reset(compiledstatement);
                }

    }

}

else
    NSLog(@"error");

sqlite3_close(database);

3 个答案:

答案 0 :(得分：6)

是否可以重构代码，以便在每次迭代时都不打开数据库？

  • 打开数据库
  • 开始交易sqlite3_exec（...，“BEGIN”，...）
  • 编译语句
  • 迭代数据集
    • 插入记录
  • 敲定已编译的陈述
  • 提交事务sqlite3_exec（...，{“ROLLBACK”或“COMMIT”}，...）
  • 关闭数据库

这与你现在的情况形成鲜明对比

  • 迭代数据集
    • 打开数据库
    • 编译语句
    • 插入记录
    • 敲定已编译的陈述
    • 关闭数据库

以您的方式执行此操作的开销会影响性能。尝试重构上面概述的方法，看看你是怎么做的。

修改

我已重新格式化您的代码以指出我在说什么。另外，我认为你所采取的其他性能（如另一个用户所示）是JSON调用。这可能是真的让你如此放慢速度。

dbPath=[self.databasePath UTF8String];
if(sqlite3_open(dbPath,&database)==SQLITE_OK)
{
    sqlite3_exec(database, "BEGIN", 0, 0, 0);
    const char *sqlstatement="insert into artist values(?,?,?,?,?)";
    sqlite3_stmt *compiledstatement;

    if(sqlite3_prepare_v2(database,sqlstatement , -1, &compiledstatement, NULL)==SQLITE_OK)
    {
        int hasError= 0;
        for(i=4611; hasError == 0 && i<=34803; i++)
        {  
            NSURLResponse *response;
            NSError *err;
            NSData *data= [NSURLConnection sendSynchronousRequest:[NSURLRequest requestWithURL:[NSURL URLWithString:[NSString stringWithFormat:@"API&id=%i",i]]] returningResponse:&response error:&err];
            if(data.length>0)
            {
                NSDictionary *jsonDict=[NSJSONSerialization JSONObjectWithData:data options:NSJSONReadingAllowFragments error:&err];
                // sqlite3_exec(database, "BEGIN", 0, 0, 0);
                sqlite3_bind_text(compiledstatement,1,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"id"] UTF8String], -1, SQLITE_TRANSIENT);
                sqlite3_bind_text(compiledstatement,2,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"name"] UTF8String], -1, SQLITE_TRANSIENT);
                if([[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"])
                    sqlite3_bind_text(compiledstatement,3,[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"description"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,3,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"])
                    sqlite3_bind_text(compiledstatement,4,[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"links"]objectForKey:@"website"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,4,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if([[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0]objectForKey:@"url"])
                    sqlite3_bind_text(compiledstatement,5,[[[[[[[jsonDict objectForKey:@"artistDetail"]objectForKey:@"artist"]objectForKey:@"media"]objectForKey:@"low_res_images"]objectAtIndex:0] objectForKey:@"url"] UTF8String], -1, SQLITE_TRANSIENT);
                else
                    sqlite3_bind_text(compiledstatement,5,[@"" UTF8String], -1, SQLITE_TRANSIENT);
                if(sqlite3_step(compiledstatement)==SQLITE_DONE)
                {
                    NSLog(@"done %i",i);
                }
                else {
                    NSLog(@"ERROR");
                    hasError= 1;
                }
            }
            sqlite3_reset(compiledstatement);
        }
        // Really need to check error conditions with commit/rollback
        if( hasError == 0 ) {
            sqlite3_exec(database, "COMMIT", 0, 0, 0);
        }
        else {
            sqlite3_exec(database, "ROLLBACK", 0, 0, 0);
        }
    }
    sqlite3_close(database);
}
else {
    NSLog(@"error");
}

答案 1 :(得分：0)

您可以执行以下操作，

  • 使用[NSSting StringWithFormat:@"Insert Statement with Parameters"]

    • 创建查询

  • 将查询存储到数组中。

  • 创建交易。您可以通过SQL Query

    • 执行此操作

  • 循环数组并执行查询。

  • 提交交易

对于每个insert语句，sqlite开始一个事务并提交它。那很重。为了避免这种开销，我们可以开始我们的交易。它非常快。

答案 2 :(得分：0)

最大的瓶子是api电话。

最佳实践解决方案是，在NSOperationQueue中将呼叫与NSOperation分开。

相关问题
最新问题