python对大文件的处理

多线程框架中采取queue来实现线程间资源的互斥。

在文件过大的情况下，如果都读入内存的话，占用内存就太多了。

这里手动实现了一个多线程调用文件迭代器来使用f.next()

# -*- coding:utf-8 -*-

import threading

class Geturl(object):

    def __init__(self, open_file):

        self.open_file = open_file

        self.num = 0

        self.__mutex = threading.RLock()

        self.f = open(self.open_file, 'r')

        self.kafka_mutex = threading.RLock()

    def _line(self):

        self.__mutex.acquire()

        try:

            line = self.f.__next__()

        except StopIteration:

            line = StopIteration

            self.f.close()

        '''

        if self.num % 1000 == 0:

            print(self.num)

        self.num += 1

        '''

        self.__mutex.release()

        return line

    def get_line(self):

        return self._line()

def _deal(deal_file):

    while True:

        try:

            item = deal_file.get_line()

            # 处理工作

            if item == StopIteration:

                raise ValueError()

        except ValueError:

            print("all task has done!")

            break

        except Exception as e:

            print("error:", e)

if __name__ == "__main__":

    filename = 'Bigfile'

    geturl = Geturl(filename)

    _deal(geturl)

巴特西

python对大文件的处理

最新文章

热门文章