在文件创建的情况下并行化事件通知

时间:2017-02-23 14:51:05

标签: file parallel-processing python-multithreading inotify

我正在使用" Inotify"在目录中创建文件或文件夹时记录事件(此处为tmp)。这里的示例以串行进程的形式完成工作。意思是,所有文件创建都以顺序方式一个接一个地处理。

import logging

import inotify.adapters

_DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

_LOGGER = logging.getLogger(__name__)

def _configure_logging():
    _LOGGER.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()

    formatter = logging.Formatter(_DEFAULT_LOG_FORMAT)
    ch.setFormatter(formatter)

    _LOGGER.addHandler(ch)

def _main():
    i = inotify.adapters.Inotify()

    i.add_watch(b'/tmp')

    try:
        for event in i.event_gen():
            if event is not None:
                (header, type_names, watch_path, filename) = event
                _LOGGER.info("WD=(%d) MASK=(%d) COOKIE=(%d) LEN=(%d) MASK->NAMES=%s "
                             "WATCH-PATH=[%s] FILENAME=[%s]",
                             header.wd, header.mask, header.cookie, header.len, type_names,
                             watch_path.decode('utf-8'), filename.decode('utf-8'))
    finally:
        i.remove_watch(b'/tmp')

if __name__ == '__main__':
    _configure_logging()
    _main()

我想引入事件通知的并行化,以防通过导入线程上传多个文件,我应该添加线程作为循环吗? 第二个问题,我不确定将线程功能放在哪里会感觉到。

1 个答案:

答案 0 :(得分:-1)

以下脚本在多个会话的情况下处理多个事件。所以在我的情况下,这就足够了。我添加了多处理选项而不是线程。我发现多处理比线程更快。

 import logging
 import threading
 import inotify.adapters
 import multiprocessing  

 _DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

 _LOGGER = logging.getLogger(__name__)

 def _configure_logging():
     _LOGGER.setLevel(logging.DEBUG)

     ch = logging.StreamHandler()

     formatter = logging.Formatter(_DEFAULT_LOG_FORMAT)
     ch.setFormatter(formatter)

     _LOGGER.addHandler(ch)



 def PopUpMessage (event):
     if event is not None:
         (header, type_names, watch_path, filename) = event
         _LOGGER.info("WD=(%d) MASK=(%d) COOKIE=(%d) LEN=(%d) MASK->NAMES=%s "
             "WATCH-PATH=[%s] FILENAME=[%s]",
             header.wd, header.mask, header.cookie, header.len, type_names,
             watch_path.decode('utf-8'), filename.decode('utf-8'))


 def My_main(count):
     i = inotify.adapters.Inotify()
     DirWatcher=i.add_watch(b'/PARA')
     try:
         while True: 
             for event in i.event_gen():
                 m = multiprocessing.Process(target=PopUpMessage, args=(event,))
                 m.start()            

     finally:
         i.remove_watch(b'/PARA')

 if __name__ == '__main__':
     _configure_logging()
     N = multiprocessing.Process(target=My_main)
     N.start()