Question

我出于性能原因在Go中的Google AppEngine项目中编写了一个模块，但需要能够从数据存储区中的某些实体中读取。我写了Go代码，以便能够读取我在Python中构建的实体，但是我收到以下错误：

datastore: flattening nested structs leads to a slice of slices: field "Messages"

Python中的模型定义：

class ModelB(ndb.Model):
    msg_id = ndb.StringProperty(indexed=False)
    cat_ids = ndb.StringProperty(repeated=True, indexed=False)
    list_ids = ndb.StringProperty(repeated=True, indexed=False)
    default_list_id_index = ndb.IntegerProperty(indexed=False)

class ModelA(ndb.Model):
    date_join = ndb.DateTimeProperty(auto_now_add=True)
    name = ndb.StringProperty()
    owner_salutation = ndb.StringProperty(indexed=False)
    owner_email_address = ndb.StringProperty()
    logo_url = ndb.StringProperty(indexed=False)
    ...
    messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True)

在Go：

type ModelB struct {
    MessageID          string   `datastore:"msg_id,noindex"`
    CategoryIDs        []string `datastore:"cat_ids,noindex"`
    ListIDs            []string `datastore:"list_ids,noindex"`
    DefaultListIDIndex int      `datastore:"default_list_id_index,noindex"`
}

type ModelA struct {
    DateJoin          time.Time `datastore:"date_join,"`
    Name              string    `datastore:"name,"`
    OwnerSalutation   string    `datastore:"owner_salutation,noindex"`
    OwnerEmailAddress string    `datastore:"owner_email_address,"`
    LogoURL           string    `datastore:"logo_url,noindex"`
    Messages          []ModelB  `datastore:"bm,"`
}

我在这里做错了吗？只是Go与Python模型定义之间的功能不兼容吗？

尝试解码ModelB

按如下方式重新定义ModelA：

import pb "appengine_internal/datastore"
import proto "code.google.com/p/goprotobuf/proto"

type ModelA struct {
    DateJoin          time.Time `datastore:"date_join,"`
    Name              string    `datastore:"name,"`
    OwnerSalutation   string    `datastore:"owner_salutation,noindex"`
    OwnerEmailAddress string    `datastore:"owner_email_address,"`
    LogoURL           string    `datastore:"logo_url,noindex"`
    Messages          []ModelB  `datastore:"-"`
}

// Load is implemented for the PropertyLoaderSaver interface.

func (seller *ModelA) Load(c <-chan datastore.Property) error {
  f := make(chan datastore.Property, 100)
  for p := range c {
    if p.Name == "bm" {
      var val pb.EntityProto
      err := proto.Unmarshal([]byte(p.Value.(string)), &val)
      if err != nil {
        return err
      }
      //TODO: Store result as a new ModelB
    } else {
      f <- p
    }
  }
  close(f)
  return datastore.LoadStruct(seller, f)
}

但是我收到以下错误： proto: required field "{Unknown}" not set

Answer 1

Go数据存储区包不支持这样的两层切片。只要[]ModelB不包含任何切片，您就可以拥有ModelB。或者，您可以在ModelB中使用ModelA，ModelB可以包含切片。但是，您不能让[]ModelB和ModelB都有切片。有关错误情况，请参阅the code。你的选择：

不要在Go
编写自己的数据存储区解串器来处理这种情况 - 这可能很难
更改python数据结构以满足Go要求并重写数据

Answer 2

我想如果你挖得够多，你会找到答案：

首先，在Python中定义LocalStructuredProperty属性时，需要设置keep_keys=True

class ModelB(ndb.Model):
    msg_id = ndb.StringProperty(indexed=False)
    cat_ids = ndb.StringProperty(repeated=True, indexed=False)
    list_ids = ndb.StringProperty(repeated=True, indexed=False)
    default_list_id_index = ndb.IntegerProperty(indexed=False)

class ModelA(ndb.Model):
    date_join = ndb.DateTimeProperty(auto_now_add=True)
    name = ndb.StringProperty()
    owner_salutation = ndb.StringProperty(indexed=False)
    owner_email_address = ndb.StringProperty()
    logo_url = ndb.StringProperty(indexed=False)
    ...
    messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True, keep_keys=True)

在我的代码中进行简单的重新定义，并在每个实体上执行put()的实体上进行映射。

然后在我的Go Code中：

type ModelB struct {
    MessageID          string   `datastore:"msg_id,noindex"`
    CategoryIDs        []string `datastore:"cat_ids,noindex"`
    ListIDs            []string `datastore:"list_ids,noindex"`
    DefaultListIDIndex int      `datastore:"default_list_id_index,noindex"`
}

type ModelA struct {
    DateJoin          time.Time `datastore:"date_join,"`
    Name              string    `datastore:"name,"`
    OwnerSalutation   string    `datastore:"owner_salutation,noindex"`
    OwnerEmailAddress string    `datastore:"owner_email_address,"`
    LogoURL           string    `datastore:"logo_url,noindex"`
    Messages          []ModelB  `datastore:"-"`
}

// Load is implemented for the PropertyLoaderSaver interface.
func (s *ModelA) Load(c <-chan datastore.Property) (err error) {
    f := make(chan datastore.Property, 32)
    errc := make(chan error, 1)
    defer func() {
        if err == nil {
            err = <-errc
        }
    }()
    go func() {
        defer close(f)
        for p := range c {
            if p.Name == "bm" {
                var b ModelB
                err := loadLocalStructuredProperty(&b, []byte(p.Value.(string)))
                if err != nil {
                    errc <- err
                    return
                }
                s.Messages = append(s.Messages, b)
            } else {
                f <- p
            }
        }
        errc <- nil
    }()
    return datastore.LoadStruct(s, f)
}

由于未导出关键功能，我不得不从appengine/datastore包中复制一堆，为了简化我需要复制的代码量，我放弃了对Reference类型的支持。我在问题跟踪器上打开了一张故障单，看看我们是否可以导出loadEntity函数：https://code.google.com/p/googleappengine/issues/detail?id=10426

import (    
    "errors"    
    "time"    

    "appengine"    
    "appengine/datastore"        

    pb "appengine_internal/datastore"    
    proto "code.google.com/p/goprotobuf/proto"    
)    

func loadLocalStructuredProperty(dst interface{}, raw_proto []byte) error {    
    var val pb.EntityProto    
    err := proto.Unmarshal(raw_proto, &val)    
    if err != nil {    
        return err    
    }    
    return loadEntity(dst, &val)    
}

//Copied from appengine/datastore since its not exported

// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
func loadEntity(dst interface{}, src *pb.EntityProto) (err error) {
c := make(chan datastore.Property, 32)
 errc := make(chan error, 1)
 defer func() {
    if err == nil {
            err = <-errc
        }
    }()
    go protoToProperties(c, errc, src)
    if e, ok := dst.(datastore.PropertyLoadSaver); ok {
        return e.Load(c)
    }
    return datastore.LoadStruct(dst, c)
}

func protoToProperties(dst chan<- datastore.Property, errc chan<- error, src *pb.EntityProto) {
    defer close(dst)
    props, rawProps := src.Property, src.RawProperty
    for {
        var (
            x       *pb.Property
            noIndex bool
        )
        if len(props) > 0 {
            x, props = props[0], props[1:]
        } else if len(rawProps) > 0 {
            x, rawProps = rawProps[0], rawProps[1:]
            noIndex = true
        } else {
            break
        }

        var value interface{}
        if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE {
            value = indexValue{x.Value}
        } else {
            var err error
            value, err = propValue(x.Value, x.GetMeaning())
            if err != nil {
                errc <- err
                return
            }
        }
        dst <- datastore.Property{
            Name:     x.GetName(),
            Value:    value,
            NoIndex:  noIndex,
            Multiple: x.GetMultiple(),
        }
    }
    errc <- nil
}

func fromUnixMicro(t int64) time.Time {
    return time.Unix(t/1e6, (t%1e6)*1e3)
}

// propValue returns a Go value that combines the raw PropertyValue with a
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
    switch {
    case v.Int64Value != nil:
        if m == pb.Property_GD_WHEN {
            return fromUnixMicro(*v.Int64Value), nil
        } else {
            return *v.Int64Value, nil
        }
    case v.BooleanValue != nil:
        return *v.BooleanValue, nil
    case v.StringValue != nil:
        if m == pb.Property_BLOB {
            return []byte(*v.StringValue), nil
        } else if m == pb.Property_BLOBKEY {
            return appengine.BlobKey(*v.StringValue), nil
        } else {
            return *v.StringValue, nil
        }
    case v.DoubleValue != nil:
        return *v.DoubleValue, nil
    case v.Referencevalue != nil:
        return nil, errors.New("Not Implemented!")
    }
    return nil, nil
}

// indexValue is a Property value that is created when entities are loaded from
// an index, such as from a projection query.
//
// Such Property values do not contain all of the metadata required to be
// faithfully represented as a Go value, and are instead represented as an
// opaque indexValue. Load the properties into a concrete struct type (e.g. by
// passing a struct pointer to Iterator.Next) to reconstruct actual Go values
// of type int, string, time.Time, etc.
type indexValue struct {
    value *pb.PropertyValue
}

Answer 3

someone1的解决方案运行良好，但我有数百万个实体，并且不想重新放入它们（将keep_keys = True添加到LocalStructuredProperty）。

所以，我创建了一个rvm use 2.1.3的缩减版本，它消除了对密钥和＆amp;的依赖性。路径等...只需用EntityProto替换pb.EntityProto，现有的python编写的实体应加载OK（我使用PropertyLoadSaver作为嵌套结构）。

免责声明：我只是使用它来阅读Go - 我还没有尝试过编写相同的实体，看看它们是否仍在加载Python。

LocalEntityProto

从Go中的Python项目加载数据存储区实体会导致嵌套的结构切片切片错误

3 个答案: