Question

我正在尝试编写一个使用shelve来永久存储返回值的memoization库。如果我有调用其他memoized函数的memoized函数，我想知道如何正确打开shelf文件。

import shelve
import functools


def cache(filename):
    def decorating_function(user_function):
        def wrapper(*args, **kwds):
            key = str(hash(functools._make_key(args, kwds, typed=False)))
            with shelve.open(filename, writeback=True) as cache:
                if key in cache:
                    return cache[key]
                else:
                    result = user_function(*args, **kwds)
                    cache[key] = result
                    return result

        return functools.update_wrapper(wrapper, user_function)

    return decorating_function


@cache(filename='cache')
def expensive_calculation():
    print('inside function')
    return


@cache(filename='cache')
def other_expensive_calculation():
    print('outside function')
    return expensive_calculation()

other_expensive_calculation()

除非这不起作用

$ python3 shelve_test.py
outside function
Traceback (most recent call last):
  File "shelve_test.py", line 33, in <module>
    other_expensive_calculation()
  File "shelve_test.py", line 13, in wrapper
    result = user_function(*args, **kwds)
  File "shelve_test.py", line 31, in other_expensive_calculation
    return expensive_calculation()
  File "shelve_test.py", line 9, in wrapper
    with shelve.open(filename, writeback=True) as cache:
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 239, in open
    return DbfilenameShelf(filename, flag, protocol, writeback)
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 223, in __init__
    Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/dbm/__init__.py", line 94, in open
    return mod.open(file, flag, mode)
_gdbm.error: [Errno 35] Resource temporarily unavailable

您建议解决此类问题的方法。

Answer 1

不，您可能没有使用相同文件名的嵌套shelve实例。

搁置模块不支持对搁置对象的并发读/写访问。（多个同时读取访问是安全的。）当程序打开一个架子进行写入时，没有其他程序可以打开它进行读写。 Unix文件锁定可用于解决此问题，但这在Unix版本中有所不同，需要了解所使用的数据库实现。

https://docs.python.org/3/library/shelve.html#restrictions

Answer 2

您可以让装饰器维护对shelve.open返回的句柄的引用，而不是尝试嵌套调用open（正如您所发现的那样，不起作用），然后如果它存在且仍然存在打开，重新用于后续通话：

import shelve
import functools

def _check_cache(cache_, key, func, args, kwargs):
    if key in cache_:
        print("Using cached results")
        return cache_[key]
    else:
        print("No cached results, calling function")
        result = func(*args, **kwargs)
        cache_[key] = result
        return result

def cache(filename):
    def decorating_function(user_function):
        def wrapper(*args, **kwds):
            args_key = str(hash(functools._make_key(args, kwds, typed=False)))
            func_key = '.'.join([user_function.__module__, user_function.__name__])
            key = func_key + args_key
            handle_name = "{}_handle".format(filename)
            if (hasattr(cache, handle_name) and
                not hasattr(getattr(cache, handle_name).dict, "closed")
               ):
                print("Using open handle")
                return _check_cache(getattr(cache, handle_name), key, 
                                    user_function, args, kwds)
            else:
                print("Opening handle")
                with shelve.open(filename, writeback=True) as c:
                    setattr(cache, handle_name, c)  # Save a reference to the open handle
                    return _check_cache(c, key, user_function, args, kwds)

        return functools.update_wrapper(wrapper, user_function)
    return decorating_function


@cache(filename='cache')
def expensive_calculation():
    print('inside function')
    return


@cache(filename='cache')
def other_expensive_calculation():
    print('outside function')
    return expensive_calculation()

other_expensive_calculation()
print("Again")
other_expensive_calculation()

输出：

Opening handle
No cached results, calling function
outside function
Using open handle
No cached results, calling function
inside function
Again
Opening handle
Using cached results

修改

您还可以使用WeakValueDictionary来实现装饰器，它看起来更具可读性：

from weakref import WeakValueDictionary _handle_dict = WeakValueDictionary() def cache(filename): def decorating_function(user_function): def wrapper(*args, **kwds): args_key = str(hash(functools._make_key(args, kwds, typed=False))) func_key = '.'.join([user_function.__module__, user_function.__name__]) key = func_key + args_key handle_name = "{}_handle".format(filename) if handle_name in _handle_dict: print("Using open handle") return _check_cache(_handle_dict[handle_name], key, user_function, args, kwds) else: print("Opening handle") with shelve.open(filename, writeback=True) as c: _handle_dict[handle_name] = c return _check_cache(c, key, user_function, args, kwds) return functools.update_wrapper(wrapper, user_function) return decorating_function

只要没有对句柄的其他引用，它就会从字典中删除。由于我们的句柄仅在对装饰函数的最外部调用结束时超出范围，因此在句柄打开时，我们将始终在dict中有一个条目，并且在它关闭后不会立即进入。

Answer 3

您正在打开文件两次但从未实际关闭它以更新文件以用于任何用途。最后使用f.close ()。

可以用嵌套方式调用Python的shelve.open吗？

3 个答案: