197 lines
6.2 KiB
Python
197 lines
6.2 KiB
Python
#!/usr/bin/env python
|
|
# HLstatsX Community Edition - Real-time player and clan rankings and statistics
|
|
# Copyleft (L) 2008-20XX Nicholas Hastings (nshastings@gmail.com)
|
|
# http://www.hlxcommunity.com
|
|
|
|
# HLstatsX Community Edition is a continuation of
|
|
# ELstatsNEO - Real-time player and clan rankings and statistics
|
|
# Copyleft (L) 2008-20XX Malte Bayer (steam@neo-soft.org)
|
|
# http://ovrsized.neo-soft.org/
|
|
|
|
# ELstatsNEO is an very improved & enhanced - so called Ultra-Humongus Edition of HLstatsX
|
|
# HLstatsX - Real-time player and clan rankings and statistics for Half-Life 2
|
|
# http://www.hlstatsx.com/
|
|
# Copyright (C) 2005-2007 Tobias Oetzel (Tobi@hlstatsx.com)
|
|
|
|
# HLstatsX is an enhanced version of HLstats made by Simon Garner
|
|
# HLstats - Real-time player and clan rankings and statistics for Half-Life
|
|
# http://sourceforge.net/projects/hlstats/
|
|
# Copyright (C) 2001 Simon Garner
|
|
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
# For support and installation notes visit http://www.hlxcommunity.com
|
|
|
|
|
|
import os,sys
|
|
|
|
# version 1.0 alpha contributed to hlstatsx community by *XYZ*SaYnt
|
|
|
|
DBHOST="localhost"
|
|
DBNAME="your_hlstats_db"
|
|
DBUSER="your_sql_username"
|
|
DBPASS="your_sql_password"
|
|
|
|
|
|
def system(cmd):
|
|
"""
|
|
executes a system call and returns the text as a string. Returns only the first line of output.
|
|
"""
|
|
print("EXECUTING: %s"%cmd)
|
|
f = os.popen(cmd)
|
|
output = f.readlines()
|
|
f.close()
|
|
if len(output) > 0:
|
|
return output[0].replace("\n","")
|
|
else:
|
|
return ""
|
|
|
|
|
|
def fetch_geodata():
|
|
"""
|
|
Obtains the geoLiteCity raw data, resulting in geoLiteCity_Location.csv and geoLiteCity_Blocks.csv
|
|
"""
|
|
# database is updated on 1st every month, so download the file from the 1st of current month
|
|
DAT = system("date +%Y%m01")
|
|
FIL = "GeoLiteCity_%s"%DAT
|
|
FILE = FIL + ".zip"
|
|
system("rm *.csv > /dev/null")
|
|
if not os.path.exists(FILE):
|
|
system("wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/" + FILE)
|
|
system("unzip -o " + FILE)
|
|
|
|
system("mv %s/GeoLiteCity-Blocks.csv geoLiteCity_Blocks.csv"%FIL)
|
|
system("mv %s/GeoLiteCity-Location.csv geoLiteCity_Location.csv.temp"%FIL)
|
|
system("rmdir " + FIL)
|
|
system("iconv -f ISO-8859-1 -t UTF-8 geoLiteCity_Location.csv.temp > geoLiteCity_Location.csv")
|
|
|
|
return
|
|
|
|
|
|
def dump_sql(fname):
|
|
"""
|
|
Dump the new sql data into our database
|
|
"""
|
|
system("mysql -u %s -p%s -h %s %s < %s"%(DBUSER,DBPASS,DBHOST,DBNAME,fname))
|
|
|
|
|
|
def write_sql(fname):
|
|
"""
|
|
Write a file of sql commands so that our data can be imported into our database.
|
|
"""
|
|
try:
|
|
fout = open(fname,"w")
|
|
except:
|
|
print("ERROR: unable to open "+fname)
|
|
return 0
|
|
|
|
fout.write("""
|
|
DROP TABLE IF EXISTS `geoLiteCity_Blocks`;
|
|
DROP TABLE IF EXISTS `geolitecity_blocks`;
|
|
DROP TABLE IF EXISTS `geolitecity_location`;
|
|
DROP TABLE IF EXISTS `geoLiteCity_Location`;
|
|
|
|
CREATE TABLE `geoLiteCity_Blocks`
|
|
(`startIpNum` bigint(11) unsigned NOT NULL default '0',
|
|
`endIpNum` bigint(11) unsigned NOT NULL default '0',
|
|
`locId` bigint(11) unsigned NOT NULL default '0'
|
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
|
|
|
|
CREATE TABLE `geoLiteCity_Location` (
|
|
`locId` bigint(11) unsigned NOT NULL default '0',
|
|
`country` varchar(2) NOT NULL,
|
|
`region` varchar(50) default NULL,
|
|
`city` varchar(50) default NULL,
|
|
`postalCode` varchar(10) default NULL,
|
|
`latitude` decimal(14,4) default NULL,
|
|
`longitude` decimal(14,4) default NULL,
|
|
PRIMARY KEY (`locId`)
|
|
) ENGINE=MyISAM DEFAULT CHARSET=utf8;
|
|
""")
|
|
|
|
# read in the raw data
|
|
f = open("geoLiteCity_Blocks.csv")
|
|
raw = f.readlines()
|
|
f.close()
|
|
del raw[0:2]
|
|
|
|
# chunk up the raw data
|
|
chunksize = 1000
|
|
gblocks = []
|
|
chunks = len(raw)/chunksize + 1
|
|
for i in xrange(0,chunks): gblocks.append(raw[i*chunksize:(i+1)*chunksize])
|
|
print("SQL: %d data items, %d chunks"%(len(raw),chunks))
|
|
del raw
|
|
|
|
for chunk in gblocks:
|
|
if chunk:
|
|
s = "insert into `geoLiteCity_Blocks`(`startIpNum`,`endIpNum`,`locId`) values"
|
|
for l in chunk:
|
|
vals = l.replace('"',"").replace('\n',"").split(',')
|
|
s += "(%s,%s,%s),"%(vals[0],vals[1],vals[2])
|
|
s = s[0:-1] # chop the last comma
|
|
fout.write(s + ";\n");
|
|
del gblocks
|
|
|
|
|
|
|
|
|
|
f = open("geoLiteCity_Location.csv")
|
|
raw = f.readlines()
|
|
f.close()
|
|
del raw[0:2]
|
|
|
|
# chunk up the raw data
|
|
gblocks = []
|
|
chunks = len(raw)/chunksize + 1
|
|
for i in xrange(0,chunks): gblocks.append(raw[i*chunksize:(i+1)*chunksize])
|
|
print("SQL: %d data items, %d chunks"%(len(raw),chunks))
|
|
del raw
|
|
|
|
for chunk in gblocks:
|
|
if chunk:
|
|
s = "insert into `geoLiteCity_Location`(`locId`,`country`,`region`,`city`,`postalCode`,`latitude`,`longitude`) values"
|
|
for l in chunk:
|
|
vals = l.replace('"',"").replace('\n',"").split(',')
|
|
for i,v in enumerate(vals): vals[i] = v.replace("'","\\'")
|
|
s += "(%s,'%s','%s','%s','%s','%s','%s'),"%(vals[0],vals[1],vals[2],vals[3],vals[4],vals[5],vals[6])
|
|
s = s[0:-1] # chop the last comma
|
|
fout.write(s + ";\n");
|
|
del gblocks
|
|
|
|
return 1
|
|
|
|
|
|
def main():
|
|
sqlname = "geodata.sql"
|
|
print("DOWNLOADING GEO DATA....")
|
|
fetch_geodata()
|
|
print("WRITING DATABASE FILE....")
|
|
if write_sql(sqlname):
|
|
print("IMPORTING DATABASE FILE....")
|
|
dump_sql(sqlname)
|
|
else:
|
|
print("Fatal error; unable to finish.")
|
|
|
|
# clean up.
|
|
system("rm "+sqlname)
|
|
|
|
print("DONE.")
|
|
|
|
|
|
|
|
main()
|
|
|