/* Copyright (c) 2000 RIPE NCC All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of the author not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ------------------------------------------------------------------------------- Module Header Filename : updatedb.c Purpose : Update The Test Traffic DBs with RVEC(6) data Author : Manuel Valente Date : 20000719 Revised : 20020814 Rene Wilhelm: added AS path information Revised : 20021210 Florian Frotzler: added IPv6 compatibility Description : Language Version : gcc version 2.95.2 OSs Tested : Solaris 2.6, Solaris 8 and (partly) Linux Command Line : updatedb [-4] [-6] path Input Files : TTM RVEC and/or RVEC6 (traceroute measurements) files Output Files : External Programs : mysqld Problems : Should make sure that it scales up to 100 boxes To Do : Comments : requires glib library (hash tables, sorting etc) ------------------------------------------------------------------------------- */ static char const rcsid[] = "$Id: updatedb.c,v 1.8 2004/03/10 18:04:33 wilhelm Exp $"; #include "as.h" #include "common.h" #include "read_line.h" #include "digest.h" #include #include #include #include /* IPv6 address in binary form */ #define INET6_BINSTRLEN 16 /* ------------------------------------------------------------------------------- Purpose : get id of (ip or AS) path in the SQL table Comments : check hash table for (ip or as) path, if not found, insert in MySQL DB and hash table return id of new entry Input : path (ip vector, as path) , npoints in path, flag if destination reached in this route, hash table, SQL table name, field name, Mysql handle Output : numeric id of the entry in the table Returns : */ int get_pathid(char* path, int npoints, GHashTable* hash, char *table, char *field, MYSQL mysql) { struct routestruct* rsp; char query [1024]; char crc[32]; int id; /* Get crc value (MD5) for route*/ crc_calc (path,crc); /* Check if route exists */ if ((rsp = g_hash_table_lookup(hash,&crc))) { //printf ("Found crc %s for %s\n",crc,path); id = rsp->id; } else { /* Insert route in DB */ sprintf (query,"INSERT INTO %s (crc,len,%s) VALUES (\"%s\",%u,\"%s\")",table, field, crc,npoints,path); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: query %s: %s\n", query, mysql_error(&mysql)); exit (1); }; id = mysql_insert_id(&mysql); rsp = malloc(sizeof(struct routestruct)); rsp->id = id; rsp->len = npoints; strcpy (rsp->crc,crc); /* Add route to hash of routes */ g_hash_table_insert (hash, &rsp->crc, rsp); } return(id); } /* ------------------------------------------------------------------------------- Purpose : IPv6 counterpart of get_pathid() Comments : check hash table for (ip or as) path, if not found, insert in MySQL DB and hash table return id of new entry Input : path (ip vector, as path) , npoints in path, flag if destination reached in this route, hash table, SQL table name, field name, Mysql handle Output : numeric id of the entry in the table Returns : */ int get_pathid_ip6(char* path, int npoints, GHashTable* hash, char *table, char *field, MYSQL mysql) { struct routestruct* rsp; char query [4096]; char crc[32]; int id,i; char **ap, *arg[32]; char *buff, *res; char addr[INET6_BINSTRLEN]; char number[4]; /* ("0" - "255") + \0 */ buff = path; /* split the hops in path and store them into arg[] */ #if defined(USE_STRTOK) /* POSIX compliant */ ap = arg; *ap = strtok (buff, " "); while (*ap != NULL) { if (**ap != '\0') { ++ap; } *ap = strtok (NULL, " \t"); } #else /* BSD specific */ for (ap = arg; (*ap = strsep (&buff," ")) != NULL ;) { if (**ap != '\0') { ++ap; } } #endif /* * the length of the binary string in the database * is proportional to number of hops */ res = (char *) malloc (INET6_BINSTRLEN * npoints); memset(res,0,INET6_BINSTRLEN * npoints); /* * convert the hops from ASCII representation (arg[]) * to binary and fill binary string (res) */ for (i=0;iid; } else { /* Insert route in DB */ sprintf (query,"INSERT INTO %s (crc,len,%s) VALUES (\"%s\",%u,",table, field, crc,npoints); strcat (query, buff); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: query %s: %s\n", query, mysql_error(&mysql)); exit (1); }; id = mysql_insert_id(&mysql); rsp = malloc(sizeof(struct routestruct)); rsp->id = id; rsp->len = npoints; strcpy (rsp->crc,crc); /* Add route to hash of routes */ g_hash_table_insert (hash, &rsp->crc, rsp); } free(buff); return(id); } /* ------------------------------------------------------------------------------- Purpose : sort records hash Params : 1st element, 2nd element Returns : 0 Comments : used by g_slist_sort */ int sort_records (gconstpointer a,gconstpointer b) { /* Store the two elements in pointers to recstruct */ struct recstruct* pa; struct recstruct* pb; pa = (struct recstruct*)a; pb = (struct recstruct*)b; /* First sort by src */ if (pa->src < pb->src) { return (-1); }; if (pa->src > pb->src) { return (1); }; /* Then sort by dst */ if (pa->dst < pb->dst) { return (-1); }; if (pa->dst > pb->dst) { return (1); }; /* Finally, sort by timestamp */ if (pa->timestamp < pb->timestamp) { return (-1); }; if (pa->timestamp > pb->timestamp) { return (1); }; return (0); }; /* ------------------------------------------------------------------------------- Purpose : Insert data from the records hash into the DB Params : pointer to a record, pointer to full record Returns : NULL Comments : used by g_slist_foreach after list has been sorted */ void process_records (gpointer data,gpointer user_data) { /* rec contains one row of uncompressed data: rangeid, routeid, aspathid, timestamp fullrec contains one row of compressed data: rangeid, routeid, aspathid, tstart, tend, numrec */ struct recstruct* rec; struct recordstruct* fullrec; struct procstruct* proc; char query [1024]; MYSQL_RES *result; MYSQL_ROW row; MYSQL mysql; unsigned int id; unsigned int tstart; unsigned int numrec; unsigned int routeid; unsigned int aspathid; unsigned int flag; rec = (struct recstruct*)data; proc = (struct procstruct*)user_data; mysql = *proc->mysql; fullrec = (struct recordstruct*)proc->rec; id = tstart = numrec = aspathid = routeid = flag = 0; /*if (fullrec->src==0) { fullrec->src = rec->src; fullrec->dst = rec->dst; fullrec->routeid = rec->routeid; fullrec->aspathid = rec->aspathid; fullrec->tstart = rec->timestamp; fullrec->tend = rec->timestamp; fullrec->numrec = 1; return; };*/ /* Normal case: same rangeid and same routeid */ if ((fullrec->src == rec->src) && (fullrec->dst == rec->dst) && (fullrec->routeid == rec->routeid) && (fullrec->aspathid == rec->aspathid)) { fullrec->tend = rec->timestamp; fullrec->numrec++; return; }; if (fullrec->src != 0) { /* something has changed, insert current fullrec in the DB */ sprintf (query,"INSERT INTO Records (src,dst,routeid,aspathid,tstart,tend,numrec) VALUES (%u,%u,%u,%u,%u,%u,%u)",fullrec->src,fullrec->dst,fullrec->routeid,fullrec->aspathid,fullrec->tstart,fullrec->tend,fullrec->numrec); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: query %s: %s\n", query, mysql_error(&mysql)); return; }; }; if ((fullrec->src != rec->src) || (fullrec->dst != rec->dst)) { /* source or destination changed; since the list is ordered we will get here only once for each src/dst combination */ /* get last stored record between this src and dst */ sprintf (query,"SELECT id,routeid,aspathid,tstart,tend,numrec FROM Records WHERE src=%u AND dst=%u ORDER BY tstart DESC LIMIT 1",rec->src,rec->dst); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: mysql query %s: %s\n", query, mysql_error(&mysql)); return; }; if (!(result = mysql_use_result(&mysql))) { fprintf(stderr, "Error: mysql_use_result: %s\n", mysql_error(&mysql)); mysql_free_result(result); return; }; if ((row = mysql_fetch_row(result))) { id = atoi(row[0]); routeid = atoi(row[1]); aspathid = atoi(row[2]); tstart = atoi(row[3]); numrec = atoi(row[5]); if (abs(rec->timestamp - atoi(row[4]))<= TIME_LIMIT) { flag = 1; }; }; mysql_free_result(result); }; if (flag && (routeid==rec->routeid) && (aspathid == rec->aspathid)) { /* the new record is a continuation from last one stored. delete old record from the DB, it will be replaced by a new one when 'fullrec' is flushed */ sprintf (query,"DELETE FROM Records WHERE id=%u",id); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: query %s: %s\n", query, mysql_error(&mysql)); return; }; /* Set fullrec to record fetched from DB */ fullrec->src = rec->src; fullrec->dst = rec->dst; fullrec->routeid = rec->routeid; fullrec->aspathid = rec->aspathid; fullrec->tstart = tstart; fullrec->tend = rec->timestamp; fullrec->numrec = numrec+1; } else { /* routeid changed or old record is no longer valid; set fullrec to rec */ fullrec->src = rec->src; fullrec->dst = rec->dst; fullrec->routeid = rec->routeid; fullrec->aspathid = rec->aspathid; fullrec->tstart = rec->timestamp; fullrec->tend = rec->timestamp; fullrec->numrec = 1; }; return; }; /* ------------------------------------------------------------------------------- Purpose : Delete records from Records hash Params : pointer to record Returns : NULL Comments : used by g_slist_foreach */ void remove_records (gpointer data,gpointer user_data) { free (data); }; /* ------------------------------------------------------------------------------- Purpose : The important function: process data from an RVEC or RVEC6 file, store it in hashes Params : filename, ranges hash, routes hash, aspaths & asnums hashes, flag that marks the IP family Returns : Number of lines processed Comments : */ int process_file (MYSQL mysql, char* filename, GHashTable* ranges, GHashTable* routes, GHashTable* asnums, GHashTable* aspaths, int ip6flag) { GSList *records; gzFile fdz; FILE *fd; unsigned int src, dst, error_count, tt, npoints, gz, k; char *src_ip6, *dst_ip6; int t, dstreached; char buf[4096]; char vector[(MAX_IP_SIZE+1)*MAX_HOPS]; char vector_ip6[(MAX_IP6_SIZE+1)*MAX_HOPS]; char path[(MAX_AS_SIZE+1)*MAX_HOPS]; char query [1024]; char *p, *pp; int *px, *testbox_id; struct recstruct* rec; struct recordstruct* fullrec; struct procstruct *proc; char *link; records = NULL; /* Allocate memory to store fullrec, empty it */ fullrec = malloc (sizeof (struct recordstruct)); memset (fullrec,0,sizeof(struct recordstruct)); gz = 0; k = 0; pp = NULL; fd = fdz = NULL; /* Check if filename starts with RVEC[6] */ link = strrchr(filename,'/'); if (link == NULL) return (0); if (strncmp(link,"/RVEC",5) != 0) return (0); /* Check if filename ends with gz */ p = strrchr(filename, '.'); if (p && strcmp(p+1,"gz") == 0) gz = 1; if(gz) { /* use gzopen */ if((fdz = gzopen(filename, "rb")) == NULL) { fprintf(stderr, "Cannot open %s: %s\n", filename, strerror(errno)); return (0); }; } else { /* use regular open */ if((fd = fopen(filename,"rb")) == NULL) { fprintf(stderr, "Cannot open %s: %s\n", filename, strerror(errno)); return (0); }; }; error_count = 0; /* read file */ while((gz && !gzeof(fdz)) || (!gz && !feof(fd))) { if(gz) { if(gzgets(fdz, buf, sizeof(buf)) == Z_NULL) continue; } else { if(fgets(buf, sizeof(buf), fd) == NULL) continue; } /* Empty vector */ memset(vector, 0, sizeof(vector)); memset(vector_ip6, 0, sizeof(vector_ip6)); /* Parse the line */ if (!ip6flag) { t = read_line(buf,&src,&dst,&tt,&npoints,vector,path,asnums); } else { t = read_line_ip6(buf,&src_ip6,&dst_ip6,&tt,&npoints, vector_ip6, path, asnums); } /* Check if function returned 0 */ if (t > 0) { error_count++; continue; } else if (t == 0) { dstreached = TRUE; } else /* t < 0 */ { dstreached = FALSE; } k++; if (npoints==0) { /* Make vector non-empty, for MySQL ? or CRC ? */ if (!ip6flag) { strcpy(vector, " "); } else { strcpy(vector_ip6, " "); } }; /* process info, insert record */ rec = malloc (sizeof(struct recstruct)); if (!ip6flag) { /* IPv4 specific processing */ px = &src; if ((testbox_id = (int*)g_hash_table_lookup (ranges,px))==NULL) { fprintf (stderr, "IP %s not found in ranges hash (file: %s)\n", rv_inttoip(src,vector),filename); /* source IP not found, no point in processing this file */ return (0); }; rec->src = *testbox_id; px = &dst; if ((testbox_id = (int*)g_hash_table_lookup (ranges,px))==NULL) { fprintf (stderr, "IP %s not found in ranges hash (file: %s)\n", rv_inttoip(dst,vector),filename); continue; }; rec->dst = *testbox_id; rec->routeid = get_pathid(vector, npoints, routes, "Routes", "route", mysql); rec->aspathid = get_pathid(path, npoints, aspaths, "ASpaths", "path", mysql); } else { /* IPv6 specific processing */ if ((testbox_id = (int*)g_hash_table_lookup (ranges,src_ip6))==NULL) { /* source IP not found, no point in processing this file */ fprintf (stderr, "IP %s not found in ranges hash (file: %s)\n", src_ip6,filename); return (0); }; rec->src = *testbox_id; if ((testbox_id = (int*)g_hash_table_lookup (ranges,dst_ip6))==NULL) { /* destination IP not found, skip record */ fprintf (stderr, "IP %s not found in ranges hash (file: %s)\n", dst_ip6,filename); continue; }; rec->dst = *testbox_id; rec->routeid = get_pathid_ip6(vector_ip6, npoints, routes, "Routes", "route", mysql); rec->aspathid = get_pathid(path, npoints, aspaths, "ASpaths", "path", mysql); } rec->timestamp = tt; /* Add rec to list */ records = g_slist_append (records,rec); }; /* Sort list, process all records*/ records = g_slist_sort(records,sort_records); proc = malloc(sizeof(struct procstruct)); proc->mysql = &mysql; proc->rec = fullrec; g_slist_foreach(records,process_records,proc); /* flush last fullrec into the DB */ sprintf (query,"INSERT INTO Records (src,dst,routeid,aspathid,tstart,tend,numrec) VALUES (%u,%u,%u,%u,%u,%u,%u)",fullrec->src,fullrec->dst,fullrec->routeid,fullrec->aspathid,fullrec->tstart,fullrec->tend,fullrec->numrec); if (mysql_query(&mysql,query)) { fprintf(stderr, "Error: query %s: %s\n", query, mysql_error(&mysql)); exit(1); }; /* Empty list */ g_slist_foreach(records,remove_records,NULL); g_slist_free(records); records = NULL; memset (fullrec,0,sizeof(struct recordstruct)); /* Close file, report errors */ if (gz) gzclose(fdz); else fclose (fd); if (error_count > 0) { fprintf (stderr,"%s has %d errors.\n",filename,error_count); }; return (k); }; /* ------------------------------------------------------------------------------- Purpose : Main function - Initialize and process files Params : filename containing the RVEC files Returns : 0 Comments : */ int main (int argc, char *argv[]) { MYSQL mysql; MYSQL mysql_ip6; unsigned int i,n,m,k,tot,tot_ip6,timesum; char *path; GHashTable *ranges,*routes,*asnums,*aspaths; GHashTable *ranges_ip6,*routes_ip6,*asnums_ip6,*aspaths_ip6; double z; int ip4flag = 0; /* "1" means do ONLY IPv4 data */ int ip6flag = 0; /* "1" means do ONLY IPv6 data */ time_t start, end; char filename [1024]; char *files[4096]; char *files_ip6[4096]; struct tms tmsdata; clock_t usertimes, systimes; char path_boxes[50]; systimes = 0; usertimes = 0; k = tot = tot_ip6 = timesum = 0; strcpy (path_boxes,PATHBOXES); /* Check arguments */ if (argc < 2) { fprintf (stderr,"Usage: %s [-4] [-6] \n",argv[0]); exit (1); }; if (strcmp(argv[1],"-4") == 0) { ip4flag = 1; path = argv[2]; printf("Processing only IPv4 data...\n"); } else if (strcmp(argv[1],"-6") == 0) { ip6flag = 1; path = argv[2]; printf("Processing only IPv6 data...\n"); } else path = argv[1]; if (ip4flag && ip6flag) { fprintf (stderr,"IP switch type mismatch, please check your syntax!"); exit (1); }; n = m = 0; /* IPv4 and IPv6 currently use their own SQL handles which connect to the v4 or v6 SQL database. This was easiest for Florian to implement. But if we don't need the flexibility of hosting these DBs on different server hosts, we can probably streamline the code to use one MySQL handle and switch DBs as appropriate with the API */ if (!ip6flag) { /* IPv4 specific */ /* Initialization: open connection, read routes and ranges */ open_admin_connection (&mysql, MYSQL_HOST); ranges = g_hash_table_new (g_int_hash,g_int_equal); routes = g_hash_table_new (g_str_hash,g_str_equal); aspaths = g_hash_table_new (g_str_hash,g_str_equal); asnums = g_hash_table_new (g_int_hash,g_int_equal); get_ranges (ranges, path_boxes, 0); get_table (&mysql, routes, "Routes"); get_table (&mysql, aspaths, "ASpaths"); n = get_filenames (path,files,0); #ifdef DEBUG printf ("%s: Found %d IPv4 files\n",path,n); #endif } if (!ip4flag) { /* IPv6 specific */ /* Initialization: open connection, read routes and ranges */ open_admin_connection_ip6 (&mysql_ip6, MYSQL_HOST); ranges_ip6 = g_hash_table_new (g_str_hash,g_str_equal); routes_ip6 = g_hash_table_new (g_str_hash,g_str_equal); aspaths_ip6 = g_hash_table_new (g_str_hash,g_str_equal); asnums_ip6 = g_hash_table_new (g_str_hash,g_str_equal); get_ranges (ranges_ip6, path_boxes, 1); get_table (&mysql_ip6, routes_ip6, "Routes"); get_table (&mysql_ip6, aspaths_ip6, "ASpaths"); m = get_filenames (path,files_ip6,1); #ifdef DEBUG printf ("%s: Found %d IPv6 files\n",path,m); #endif } /* Connect to Routing Registry or compatible RISwhois server */ /* To not keep the connection lingering, we first wait until the (timeconsuming) get_ranges and get_table have completed */ RRopen(RR_SERVER, RR_SERVICE); RRconnect(); /* For each IPv4 file */ for (i=0;i < n; i++) { start = time(NULL); strcpy (filename,files[i]); /* Process this file */ if ((k = process_file(mysql,filename,ranges,routes, asnums,aspaths,0)) == 0) { continue; } tot += k; end = time(NULL); timesum += (end -start); z = (end - start) ? k/(end - start) : k; times (&tmsdata); usertimes = tmsdata.tms_utime - usertimes; systimes = tmsdata.tms_stime - systimes; /* printf ("%s %u %lu %4.2f %lu %lu\n",files[i], k, (end - start), z, usertimes, systimes); */ usertimes = tmsdata.tms_utime; systimes = tmsdata.tms_stime; }; /* For each IPv6 file */ for (i=0;i < m; i++) { start = time(NULL); strcpy (filename,files_ip6[i]); /* Process this file */ if ((k = process_file(mysql_ip6,filename,ranges_ip6,routes_ip6, asnums_ip6, aspaths_ip6, 1)) == 0) { continue; } tot_ip6 += k; end = time(NULL); timesum += (end -start); z = (end - start) ? k/(end - start) : k; times (&tmsdata); usertimes = tmsdata.tms_utime - usertimes; systimes = tmsdata.tms_stime - systimes; /* printf ("%s %u %lu %4.2f %lu %lu\n",files[i], k, (end - start), z, usertimes, systimes); */ usertimes = tmsdata.tms_utime; systimes = tmsdata.tms_stime; }; if (!ip6flag && (tot == 0)) { printf ("No IPv4 files could be processed.\nPlease check that you have used full path filenames.\n"); } if (!ip4flag && (tot_ip6 == 0)) { printf ("No IPv6 files could be processed.\nPlease check that you have used full path filenames.\n"); } printf ("%u IPv4 files (%u lines) and %u IPv6 files (%u lines) processed in %u seconds\n",n,tot,m,tot_ip6,timesum); if (!ip6flag) { mysql_close(&mysql); } if (!ip4flag) { mysql_close(&mysql_ip6); } exit (0); };