Question

我正在编写另一个python清除脚本。这是用大量的find -delete替换一个非常旧的bash脚本，这需要花费9h才能清除我们的视频后端。

我知道谷歌有很多或者谷歌，但是我有更多的约束条件让我写下我发现的差/无效代码。

考虑以下目录结构：

/data/channel1/video_800/0001/somefile_800_001.ts
  /data/channel1/video_800/0001/somefile_800_002.ts
  /data/channel1/video_800/0002/somediffile_800_001.ts
  /data/channel1/video_800/0002/somediffile_800_002.ts
  /data/channel1/video_800.m3u8
  /data/channel1/video_900/0001/someotherfile_900_001.ts
  /data/channel1/video_900/0002/afile_900_001.ts
  /data/channel1/video_900/0003/bfile_900_001.ts
  /data/channel1/video_900/0003/cfile_900_001.ts
  /data/channel1/video_900.m3u8

/data/channel2/video_800/0001/againsomefile_800_001.ts
  /data/channel2/video_800/0001/againsomefile_800_001.ts
  /data/channel2/video_800.m3u8

/data/sport_channel/video_1000/0001/somefile.ts   /data/sport_channel/video_1000/0001/somefile2.ts

我感兴趣的第一件事是频道名称，因为频道*有一条规则，而运动*则有。

第二件事是视频dirs的结尾等于比特率... 800,900,1000，因为它们可以有不同的保留天数。

最后我会查看所有内容并根据比特率和扩展名删除文件。

波纹管代码有效，但过于复杂，我确定不是非常pythonic。因为在这种情况下我最关心的是性能，我确信有更有效的方法来做到这一点。在for循环中堆叠for循环不仅设计不佳，而且还得到了一个＆＃39; find_files＆＃39;在我的pymode中过于复杂[mccabe] 。

**将删除函数从代码示例中删除，但它只是一个简单的尝试：除了使用os.rmdir和os.remove

我愿意接受改进我的代码的所有建议。

谢谢！

#!/usr/bin/python

import os
import time
import fnmatch

path = '/data'
debits_short = ['200', '700', '1000', '1300', '2500']
debits_long = ['400', '1800']

def find_files(chan_name, debits, duration):

    time_in_secs = time.time() - (duration * 24 * 60 * 60)

    # List channel
    for channel in os.listdir(path):

        # Match category channels
        if fnmatch.fnmatch(channel, chan_name):

            # Go through bitrates
            for debit in debits:

                # Channel path now the default search path
                channel_path = path + channel

                # Walk through channel path to match bitrate files
                for root, dirs, files in os.walk(channel_path, topdown=False):
                    for filename in files:

                        # Remove files that contain _bitrate_ and end with ts
                        if '_' + debit + '_' in filename:
                            if filename.endswith('.ts'):
                                if os.path.isfile(os.path.join(root, filename)):
                                    if os.stat(os.path.join(root, filename)).st_mtime <= time_in_secs:
                                        remove(os.path.join(root, filename))

                        # Remove playlist files that contain bitrate.m3u8
                        if filename.endswith(debit + '.m3u8'):
                            if os.path.isfile(os.path.join(root, filename)):
                                if os.stat(os.path.join(root, filename)).st_mtime <= time_in_secs:
                                    remove(os.path.join(root, filename))

                    # Remove empty dirs
                    for dir in dirs:
                        if not os.listdir(os.path.join(root, dir)):
                            remove(os.path.join(root, dir))


find_files('channel*', debits_long, 3)
find_files('sport*', debits_short, 7)

Answer 1

这是一种可行的方法：

import os
import glob
import time


class Purge(object):

    removable_extensions = ['ts', 'm3u8']

    def __init__(self, basedir, channel_pattern, debits,
                 older_than_days, test_mode=False):
        self.basedir = basedir
        self.channel_pattern = channel_pattern
        self.debits = debits
        self.older_than_secs = time.time() - 24*60*60*older_than_days
        self.test_mode = test_mode  # If `True`, do not delete files.

    def delete_file(self, filepath):
        try:
            os.remove(filepath)
        except OSError:
            pass

    def file_for_deletion(self, filepath):
        # Return `True` if a file meets all conditions for deletion.
        filename, ext = os.path.splitext(os.path.basename(filepath))
        condition_ext = ext[1:] in self.removable_extensions
        condition_old = os.stat(filepath).st_mtime <= self.older_than_secs
        condition_deb = any(
            '_{}_'.format(d) in filename or filename.endswith(d)
            for d in self.debits
            )
        return all((condition_ext, condition_old, condition_deb))

    def purge_channel(self, channel_dir):
        for root, dirs, files in os.walk(channel_dir):
            for name in files:
                filepath = os.path.join(root, name)
                if self.file_for_deletion(filepath):
                    print filepath
                    if not self.test_mode:
                        self.delete_file(filepath)
            #TODO: delete empty directories here.

    def purge(self):
        channels = glob.glob(os.path.join(self.basedir, self.channel_pattern))
        for channel_dir in channels:
            self.purge_channel(channel_dir)


if __name__ == '__main__':

    purge_job_info = dict(
        basedir=r'path/to/data',  # All channel folders live here.
        channel_pattern='channel*',  # `glob` pattern.
        debits=['400', '1800'],
        older_than_days=7,
        )

    p = Purge(**purge_job_info)
    p.test_mode = True
    p.purge()

清除具有多种条件的脚本

1 个答案: