# Author : Fajri Abdillah a.k.a clasense4 # Twitter : @clasense4 # mail : [email protected] # import modules used here -- sys is a very standard one import sys import redis from datetime import datetime import time import MySQLdb # connect to the MySQL server try: conn = MySQLdb.connect ( host = "localhost", user = "root", passwd = "testroot", db = "django_crawler" ) except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) sys.exit (1) # START TIME startTime = datetime.now() # CURSOR DB CURSOR = conn.cursor() # Redis Object R_SERVER = redis.Redis("localhost") sql = "select link_href from cg_news" # Create a key key = "link:rss2" print "Created Key\t\t : %s" % key def insert_redis(sql): # INPUT 1 : SQL query # OUTPUT : Array of result # Do MySQL query CURSOR.execute(sql) data = CURSOR.fetchall() print sql print "MySQLexecution time\t : %s" % str(datetime.now()-startTime) # timer inserting with redis start_time_redis = datetime.now() counter = 0 count_data = 0 for datas in data : # Check if data exists in set. if (R_SERVER.sadd(key, datas)): #print "%s " % (datas) counter += 1 count_data += 1 print "Redis Execution time\t : %s " % (str(datetime.now()-start_time_redis)) print "Inserted data\t\t : %s" % (counter) print "count data\t\t : %s" % (count_data) CURSOR.close() # Standard boilerplate to call the main() function to begin # the program. if __name__ == '__main__': insert_redis(sql)