我在我的代码中遇到了内存错误。我的解析器可以概括为:
# coding=utf-8
#! /usr/bin/env python
import sys
import json
from collections import defaultdict
class MyParserIter(object):
def _parse_line(self, line):
for couple in line.split(","):
key, value = couple.split(':')[0], couple.split(':')[1]
self.__hash[key].append(value)
def __init__(self, line):
# not the real parsing just a example to parse each
# line to a dict-like obj
self.__hash = defaultdict(list)
self._parse_line(line)
def __iter__(self):
return iter(self.__hash.values())
def to_dict(self):
return self.__hash
def __getitem__(self, item):
return self.__hash[item]
def free(self, item):
self.__hash[item] = None
def free_all(self):
for k in self.__hash:
self.free(k)
def to_json(self):
return json.dumps(self.to_dict())
def parse_file(file_path):
list_result = []
with open(file_path) as fin:
for line in fin:
parsed_line_obj = MyParserIter(line)
list_result.append(parsed_line_obj)
return list_result
def write_to_file(list_obj):
with open("out.out", "w") as fout:
for obj in list_obj:
json_out = obj.to_json()
fout.write(json_out + "\n")
obj.free_all()
obj = None
if __name__ == '__main__':
result_list = parse_file('test.in')
print(sys.getsizeof(result_list))
write_to_file(result_list)
print(sys.getsizeof(result_list))
# the same result for memory usage result_list
print(sys.getsizeof([None] * len(result_list)))
# the result is not the same :(
目标是解析(大型)文件,将每行转换为JSON对象并写回到文件中。
我的目标是减少占用空间,因为在某些情况下,此代码会引发内存错误。在每次fout.write
之后,我希望删除(释放内存)obj
引用。
我尝试将obj
设置为None或调用obj.free_all()
方法,但它们都没有释放内存。我也使用了simplejson而不是json,这减少了占用空间,但在某些情况下仍然太大。
test.in看起来像:test1:OK,test3:OK,...
test1:OK,test3:OK,...
test1:OK,test3:OK,test4:test_again...
....