测试环境为阿里云，单核Intel(R) Xeon(R) CPU E5-2420 0 @ 1.90GHz，内存512M，最基础的配置，测试系统为CentOS 64位。测试数据就是把双色球所有组合进行排列，共17721088行，文件名为di.txt,体积363421k。leveldb推荐使用SSD硬盘，当前虚拟硬盘的速度肯定是不行，此处只是学习，测试对比。
官方网址
http://code.google.com/p/py-leveldb/
安装命令
svn checkout http://py-leveldb.googlecode.com/svn/trunk/ py-leveldb-read-only
cd py-leveldb-read-only/
# 需要注意，下面的脚本里需要使用git获取leveldb。所以要把git客户端装好。
./compile_leveldb.sh
python setup.py build
python setup.py install
python遇到的问题
报错
../snappy-read-only/snappy.h:45:33: error: snappy-stubs-public.h
解决：
yum -y install autoconf automake libtool
再次编译还是不成功，手动安装压缩工具。
网址：http://code.google.com/p/snappy/
使用命令
wget http://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
./configure –enable-shared=no –enable-static=yes
make CXXFLAGS=’-g -O2 -fPIC’
make install
再次安装编译py-leveldb通过。
C语言编译问题
报错
/usr/bin/ld: cannot find -lleveld
collect2: ld 返回 1
解决
将.so .a拷贝到系统lib目录中，测试系统64位，直接拷贝到lib64中。
cp libleveldb.* /usr/lib64/
python测试部分
顺序写代码
#!/bin/python
#-*- coding:utf-8 -*-
# Filename:
# Revision:
# Date: 2013-12-14
# Author: simonzhang
# web: www.simonzhang.net
# Email: simon-zzm@163.com
### END INIT INFO
import leveldb
import time
db = leveldb.LevelDB('./data')
# multiple put/delete applied atomically, and committed to disk
#batch = leveldb.WriteBatch()
f = open('di.txt')
num = 0
start = time.time()
for i in f:
if i[-1] == '\n':
data = i[:-1]
else:
data = i
num += 1
db.Put("%s" % num, data)
end = time.time()
print "use sec %s" % (end-start)
批量写代码
#!/bin/python
#-*- coding:utf-8 -*-
# Filename:
# Revision:
# Date: 2013-12-14
# Author: simonzhang
# web: www.simonzhang.net
# Email: simon-zzm@163.com
### END INIT INFO
import leveldb
import time
db = leveldb.LevelDB('./data')
batch = leveldb.WriteBatch()
f = open('di.txt')
num = 0
start = time.time()
for i in f:
if i[-1] == '\n':
data = i[:-1]
else:
data = i
num += 1
batch.Put("%s" % num, data)
# 因为内存太小，每5万行写入一次
if ((num % 50000) == 0) or (num == 17721087):
db.Write(batch, sync = True)
batch = leveldb.WriteBatch()
end = time.time()
print "use sec %s" % (end-start)
随机读1000万次代码：
#!/bin/python
#-*- coding:utf-8 -*-
# Filename:
# Revision:
# Date: 2013-12-14
# Author: simonzhang
# web: www.simonzhang.net
# Email: simon-zzm@163.com
### END INIT INFO
import leveldb
from random import randint
import time
db = leveldb.LevelDB('./data')
start = time.time()
for i in xrange(10000000):
num = randint(1, 10000000)
try:
v = db.Get("%s" % num)
print v
except:
pass
end = time.time()
print "use sec %s" % (end-start)
测试结果
# python write_seq.py
use sec 329.217786074
每秒写入53827
# python write_bacth.py
use sec 173.626176119
每秒写入102064
# python read.py
use sec 288.070755005
每秒随机读取34713
C部分代码，为了方便，我把两个代码分开写。
C顺序写入
// Filename: // Revision: // Date: 2013-12-14 // Author: simonzhang // web: www.simonzhang.net // Email: simon-zzm@163.com // END INIT INFO #include#include #include #include #include "leveldb/c.h" char *itoa(int value, char *string, int radix) { int rt=0; if(string==NULL) return NULL; if(radix<=0 || radix>30) return NULL; rt = snprintf(string, radix, "%d", value); if(rt>radix) return NULL; string[rt]='\0'; return string; } int main() { leveldb_t *db; leveldb_options_t *options; leveldb_readoptions_t *roptions; char *err = NULL; char *read; size_t read_len; time_t start_time, end_time; // write file FILE *di; if((di=fopen("di.txt","r"))==NULL) { printf("can't open!\n"); return -1; } // OPEN leveldba options = leveldb_options_create(); // leveldb_options_set_error_if_exists(options, 1); // leveldb_options_set_cache(options, cache); // leveldb_options_set_env(options, env); // leveldb_options_set_info_log(options, NULL); // leveldb_options_set_write_buffer_size(options, 100000); // leveldb_options_set_paranoid_checks(options, 1); // leveldb_options_set_max_open_files(options, 4096); // leveldb_options_set_block_size(options, 1024); // leveldb_options_set_block_restart_interval(options, 8); // leveldb_options_set_compression(options, leveldb_no_compression); leveldb_options_set_compression(options, leveldb_snappy_compression); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, "data", &err); if (err != NULL) { fprintf(stderr, "Open fail.\n"); return(1); } leveldb_free(err); err = NULL; roptions = leveldb_readoptions_create(); // start read start_time = time(NULL); int X=99,Y=15000000; //X为起始值 Y为终止值 int _rand;//随机数 char s[8]; srand( (unsigned)time( NULL ) ); for (int i = 0; i<10000000; i++) { _rand = rand()%(Y-X+1)+X; itoa(_rand,s,8); read = leveldb_get(db, roptions, s, strlen(s), &read_len, &err); //printf("%s\n", read); if (err != NULL) { fprintf(stderr, "Read fail.\n"); return(1); } leveldb_free(err); free(read); } // CLOSE` leveldb_close(db); end_time = time(NULL); printf("%ld\n", end_time-start_time); return 0; }
C 1000万次随机读
// Filename: // Revision: // Date: 2013-12-14 // Author: simonzhang // web: www.simonzhang.net // Email: simon-zzm@163.com // END INIT INFO #include#include #include #include #include "leveldb/c.h" char *itoa(int value, char *string, int radix) { int rt=0; if(string==NULL) return NULL; if(radix<=0 || radix>30) return NULL; rt = snprintf(string, radix, "%d", value); if(rt>radix) return NULL; string[rt]='\0'; return string; } int main() { leveldb_t *db; leveldb_options_t *options; leveldb_writeoptions_t *woptions; char *err = NULL; time_t start_time, end_time; // write file FILE *di; if((di=fopen("di.txt","r"))==NULL) { printf("can't open!\n"); return -1; } // OPEN leveldba options = leveldb_options_create(); // leveldb_options_set_error_if_exists(options, 1); // leveldb_options_set_cache(options, cache); // leveldb_options_set_env(options, env); // leveldb_options_set_info_log(options, NULL); // leveldb_options_set_write_buffer_size(options, 100000); // leveldb_options_set_paranoid_checks(options, 1); // leveldb_options_set_max_open_files(options, 4096); // leveldb_options_set_block_size(options, 1024); // leveldb_options_set_block_restart_interval(options, 8); // leveldb_options_set_compression(options, leveldb_no_compression); leveldb_options_set_compression(options, leveldb_snappy_compression); leveldb_options_set_create_if_missing(options, 1); db = leveldb_open(options, "data", &err); if (err != NULL) { fprintf(stderr, "Open fail.\n"); return(1); } leveldb_free(err); err = NULL; woptions = leveldb_writeoptions_create(); // start write start_time = time(NULL); char ch, str[30]; int num=0, c=0; ch = fgetc(di); while(ch!=EOF) { if (ch == '\n') { char s[10]; itoa(num,s,10); leveldb_put(db, woptions, s, strlen(s), str, strlen(str), &err); if (err != NULL) { fprintf(stderr, "Write fail.\n"); return(1); } memset(str,'\0',sizeof(str)); c = 0; num += 1; } else { str[c] = ch; c += 1; } ch = fgetc(di); } fclose(di); // CLOSE leveldb_close(db); end_time = time(NULL); printf("%ld\n", end_time-start_time); return 0; }
测试
C顺序写入
编译
gcc -Wall -std=c99 write-leveldb.c -lleveldb -O3 -o write-leveldb
结果
# ./write-leveldb
225
每秒钟处理78760
C 1000万次随机读
编译
gcc -Wall -std=c99 read-leveldb.c -lleveldb -O3 -o read-leveldb
结果
# ./read-leveldb
143
每秒处理69930
写入过程CPU肯定是全部跑满。使用snappy压缩，所以写入data目录为175M，压缩了一半。
随机读将CPU跑满。python内存占用23%。C语言占用内存最终增加到39%。
之后又做到了一个测试，硬件内存只有512M,硬盘数据插入826M。使用python代码再次随机读取1000万次，使用347.94秒，每秒随机读28740。所以数据超出物理内存不会出错只是速度下降。
还有问题一没有测试，leveldb默认的每块2M如果64G则数据文件65536个，达到系统打开文件最大数，不知道会不会出问题。并且在同一个目录下文件过多也会对系统改造成一定压力，不知道是否会有影响。推荐使用办法把单块体积加大，此效率也没有测试。
还有一点说明，使用pypy做了测试，效果不如python，具体原因没有详查。
