Question

我使用的是谷歌应用引擎，需要拥有1000到2 ^ 31之间的实体密钥。我考虑了两种方法：

1）保留所创建密钥的计数器，详见https://cloud.google.com/appengine/articles/sharding_counters。但这需要对每个密钥进行多次数据存储区读/写操作，并且我不确定它是否保证一致。

2）在我的范围内生成一个随机int并检查该密钥是否已存在于数据库中。为了使它便宜，我想要一个keys_only查询，但除了将密钥另外保存为一个单独的字段之外，我无法找到这样做的方法： MyEntity.query（MyEntity.key_field == new_random_number）.fetch（keys_only = TRUE）

有没有更好的方法来实现这一目标？

Answer 1

您期望在制作中每秒写入多少次？你的两个建议都很好，但对于我们的应用，我决定采用分片计数法。您还可以在放置实体之前设置实体的ID以完全避免查询：

MyModel(id="foo")

然后你可以查一查：

MyModel.get_by_id("foo")

我不必是一个字符串，它也可以是一个数字：

MyModel(id=123)

如果您决定使用分片计数器，请参阅我们的生产级代码，该代码与您在该文章中阅读的内容非常接近; o）Memcache会增加我们所需的一致性级别对了。

class GeneralShardedCounterConfig(ndb.Model):
    SHARD_KEY_TEMPLATE = 'gen-count-{}-{:d}'
    num_shards = ndb.IntegerProperty(default=200)

    @classmethod
    def all_keys(cls, name):
        config = cls.get_or_insert(name)
        shard_key_strings = [GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
                             for index in range(config.num_shards)]
        return [ndb.Key(GeneralShardedCounter, shard_key_string)
                for shard_key_string in shard_key_strings]


class GeneralShardedCounter(BaseModel):
    count = ndb.IntegerProperty(default=0)

    @classmethod
    def get_count(cls, name):
        total = memcache.get(name)
        if total is None:
            total = 0
            all_keys = GeneralShardedCounterConfig.all_keys(name)
            for counter in ndb.get_multi(all_keys):
                if counter is not None:
                    total += counter.count
            memcache.set(name, total, constants.SHORT_MEMCACHE_TTL)
        return total

    @classmethod
    @ndb.transactional(retries=5)
    def increase_shards(cls, name, num_shards):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        if config.num_shards < num_shards:
            config.num_shards = num_shards
            config.put()

    @classmethod
    @ndb.transactional(xg=True)
    def _increment(cls, name, num_shards):
        index = random.randint(0, num_shards - 1)
        shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
        counter = cls.get_by_id(shard_key_string)
        if counter is None:
            counter = cls(id=shard_key_string)
        counter.count += 1
        counter.put()
        # Memcache increment does nothing if the name is not a key in memcache
        memcache.incr(name)

    @classmethod
    def increment(cls, name):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        cls._increment(name, config.num_shards)

    @classmethod
    def _add(cls, name, value, num_shards):
        index = random.randint(0, num_shards - 1)
        shard_key_string = GeneralShardedCounterConfig.SHARD_KEY_TEMPLATE.format(name, index)
        counter = cls.get_by_id(shard_key_string)
        if counter is None:
            counter = cls(id=shard_key_string)
        counter.count += value
        counter.put()
        # Memcache increment does nothing if the name is not a key in memcache
        memcache.incr(name, value)

    @classmethod
    def add(cls, name, value):
        config = GeneralShardedCounterConfig.get_or_insert(name)
        cls._add(name, value, config.num_shards)

Answer 2

get_or_insert的示例。插入7个唯一键

import webapp2
from google.appengine.ext import ndb
from datetime import datetime
import random
import logging


class Examples(ndb.Model):
    data = ndb.StringProperty()
    modified = ndb.DateTimeProperty(auto_now=True)
    created = ndb.DateTimeProperty()  # NOT auto_now_add HERE !!


class MainHandler(webapp2.RequestHandler):

    def get(self):

        count = 0
        while count < 7:
            random_key = str(random.randrange(1, 9))
            dt_created = datetime.now()
            example = Examples.get_or_insert(random_key, created=dt_created, data='some data for ' + random_key)
            if example.created != dt_created:
                logging.warning('Random key %s not unique' % random_key)
                continue
            count += 1

        self.response.write('Keys inserted')

app = webapp2.WSGIApplication([
    ('/', MainHandler)
], debug=True)

ndb范围内的唯一键

2 个答案: