osmsucker-ywk.c

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <sys/mman.h>
   4 #include <sys/types.h>
   5 #include <sys/stat.h>
   6 #include <fcntl.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <math.h>
  10 #include <unistd.h>
  11
  12 /*
  13  * <osm>
  14  *  <node>
  15  *   <tag k=".." v=".." />
  16  *  </node>
  17  *  <way>
  18  *   <nd>
  19  *   <tag>
  20  *  </way>
  21  *  <relation>
  22  *   <member>
  23  *   <tag>
  24  *  </relation>
  25  */
  26
  27 /* TODO:
  28  * Do something usefull with changesets and version;
  29  */
  30
  31 #ifdef MMAP
  32 # define stopcondition (start = ++end) < (range + max)
  33 # define nextline start = end + 1
  34 #else
  35 # define nextline free(start); start = NULL; tmp = getline(&start, &tmplen, stdin); if (tmp == -1) { goto exit; }
  36 # define stopcondition 1
  37 #endif
  38
  39 #define file_delete_nodes               "delete_nodes.csv"
  40 #define file_delete_ways                "delete_ways.csv"
  41 #define file_delete_relations           "delete_relations.csv"
  42
  43 #define file_modify_nodes               "modify_nodes.csv"
  44 #define file_modify_ways                "modify_ways.csv"
  45 #define file_modify_relations           "modify_relations.csv"
  46
  47 #define file_nodes                      "nodes.csv"
  48
  49 #ifdef BENCHMARK
  50 #define file_nodes_uint                 "nodes_uint.csv"
  51 #define file_nodes_gis                  "nodes_gis.csv"
  52 #endif
  53
  54 #define file_node_tags                  "node_tags.csv"
  55 #define file_ways                       "ways.csv"
  56 #define file_way_tags                   "way_tags.csv"
  57 #define file_way_nds                    "way_nds.csv"
  58 #define file_relations                  "relations.csv"
  59 #define file_relation_tags              "relation_tags.csv"
  60 #define file_relation_member_node       "relation_member_node.csv"
  61 #define file_relation_member_relation   "relation_member_relation.csv"
  62 #define file_relation_member_way        "relation_member_way.csv"
  63
  64 /* http://www.cs.utk.edu/~vose/c-stuff/bithacks.html */
  65 static unsigned int reverse(unsigned int v) {
  66         unsigned int t = v << 1;  // t will have the reversed bits of v
  67         int i;
  68
  69         v >>= 1;
  70         for (i = sizeof(v) * 8 - 2; i; i--)
  71         {
  72           t |= v & 1;
  73           t <<= 1;
  74           v >>= 1;
  75         }
  76         t |= v;
  77
  78         return t;
  79 }
  80
  81 static unsigned long zcurve(char *lon, char *lat) {
  82         unsigned int array[2];
  83         unsigned int i,j;
  84         unsigned long out;
  85         double temp = strtod(lon, (char **) NULL) + 180.0; /* Longitude */
  86         if (temp < 0.0 || temp >= 360.0) temp = 0.0;
  87         array[0] = reverse(  5965232.0 * temp);
  88
  89         temp = strtod(lat, (char **) NULL) + 90.0; /* Latitude */
  90         if (temp < 0.0 || temp >= 180.0) temp = 0.0;
  91         array[1] = reverse( 11930464.0 * temp);
  92
  93         for (j = 0; j < sizeof(unsigned int)*8; j++) {
  94                 for (i = 0; i < 2; i++) {
  95                         out <<= 1;
  96                         out |= array[i] & 1;
  97                         array[i] >>= 1;
  98                 }
  99         }
 100
 101         return out;
 102 }
 103
 104
 105 static unsigned int coordtouint(char *input) {
 106         double maxbit = (double) 4294967296.0 / (double) 360.0;
 107         double proper = strtod(input, NULL) * maxbit;
 108         return (unsigned int) proper;
 109 }
 110
 111 static char * escape_string(char *instr)
 112 {
 113         unsigned int i, j=0, need = 0;
 114         unsigned int len = strlen(instr);
 115         char *outstr;
 116
 117         for (i=0;i<len;i++)
 118                 if (instr[i]=='\\' || instr[i]=='\'') need++;
 119
 120         len += need;
 121         outstr = malloc(len + 1);
 122
 123         for (i=0;i<=strlen(instr);i++) {
 124               if (instr[i]=='\\' || instr[i]=='\'')
 125                outstr[j++]='\\';
 126               outstr[j++]=instr[i];
 127     }
 128     return outstr;
 129 }
 130
 131 #ifdef MMAP
 132 static void parser(char *range, unsigned long int max) {
 133 #else
 134 static void parser() {
 135 #endif
 136     typedef enum { NONE = 0, CREATE = 1, MODIFY = 2, DELETE = 3 } ywk_state_t;
 137     typedef enum { OSM = 0, NODE = 1, WAY = 2, RELATION = 3, TAG = 4, ND = 5, MEMBER = 6 } osm_state_t;
 138     typedef enum { UNKNOWN = 0, ID, LAT, LON, USER, UID, TIMESTAMP, KEY, VALUE, TYPE, REF, ROLE, VERSION, CHANGESET} key_state_t;
 139     char *attr_id = NULL, *attr_lat = NULL, *attr_lon = NULL, *attr_user = NULL, *attr_uid = NULL, *attr_timestamp = NULL, *attr_changeset = NULL, *attr_key = NULL, *attr_value = NULL,
 140          *attr_type = NULL, *attr_ref = NULL, *attr_role = NULL, *attr_version = NULL;
 141
 142     unsigned int attr_lat_uint = 0;
 143     unsigned int attr_lon_uint = 0;
 144
 145
 146     unsigned long int count_nodes = 0, count_node_tags = 0,
 147     count_ways = 0, count_way_tags = 0, count_way_nds = 0,
 148     count_relations = 0, count_relation_tags = 0, count_members_node = 0, count_members_relation = 0, count_members_way = 0,
 149     count_delete_nodes = 0, count_delete_ways = 0, count_delete_relations = 0,
 150     count_modify_nodes = 0, count_modify_ways = 0, count_modify_relations = 0;
 151
 152     unsigned long int sequence = 0;
 153
 154     ywk_state_t ywk = NONE;
 155     osm_state_t current_tag = OSM;
 156     osm_state_t parent_tag = OSM;
 157
 158     char *start = NULL, *end, *nodename, *nodename_end, *nodename_fast_end;
 159     ssize_t tmp;
 160     size_t tmplen = 0;
 161
 162 #ifdef MMAP
 163     start = range;
 164 #else
 165     nextline;
 166 #endif
 167     end = strchrnul((const char*) start, '\n');
 168
 169     if (strncmp(start, "<?xml", 5) != 0)
 170         return;
 171
 172     nextline;
 173     end = strchrnul((const char*) start, '\n');
 174
 175     if (strncmp(start, "<osm", 4) != 0)
 176         return;
 177
 178     if (strncmp(&start[4], "Change", 6) == 0)
 179         ywk = CREATE;
 180
 181 FILE *fd_nodes,
 182 #ifdef BENCHMARK
 183     *fd_nodes_uint,
 184     *fd_nodes_gis,
 185 #endif
 186     *fd_node_tags,
 187     *fd_ways,
 188     *fd_way_tags,
 189     *fd_way_nds,
 190     *fd_relations,
 191     *fd_relation_tags,
 192     *fd_members_node,
 193     *fd_members_relation,
 194     *fd_members_way,
 195     *fd_delete_nodes,
 196     *fd_delete_ways,
 197     *fd_delete_relations,
 198     *fd_modify_nodes,
 199     *fd_modify_ways,
 200     *fd_modify_relations
 201     ;
 202
 203     fd_nodes = fopen(file_nodes, "w");
 204     if (fd_nodes == NULL) { perror("Open:"); exit(-1); }
 205 #ifdef BENCHMARK
 206     fd_nodes_uint = fopen(file_nodes_uint, "w");
 207     if (fd_nodes_uint == NULL) { perror("Open:"); exit(-1); }
 208     fd_nodes_gis = fopen(file_nodes_gis, "w");
 209     if (fd_nodes_gis == NULL) { perror("Open:"); exit(-1); }
 210 #endif
 211     fd_node_tags = fopen(file_node_tags, "w");
 212     if (fd_node_tags == NULL) { perror("Open:"); exit(-1); }
 213    fd_ways = fopen(file_ways, "w");
 214    if (fd_ways == NULL) { perror("Open:"); exit(-1); }
 215         fd_way_tags = fopen(file_way_tags, "w");
 216         if (fd_way_tags == NULL) { perror("Open:"); exit(-1); }
 217         fd_way_nds = fopen(file_way_nds, "w");
 218         if (fd_way_nds == NULL) { perror("Open:"); exit(-1); }
 219         fd_relations = fopen(file_relations, "w");
 220         if (fd_relations == NULL) { perror("Open:"); exit(-1); }
 221         fd_relation_tags = fopen(file_relation_tags, "w");
 222         if (fd_relation_tags == NULL) { perror("Open:"); exit(-1); }
 223         fd_members_node = fopen(file_relation_member_node, "w");
 224         if (fd_members_node == NULL) { perror("Open:"); exit(-1); }
 225         fd_members_relation = fopen(file_relation_member_relation, "w");
 226         if (fd_members_relation == NULL) { perror("Open:"); exit(-1); }
 227         fd_members_way = fopen(file_relation_member_way, "w");
 228         if (fd_members_way == NULL) { perror("Open:"); exit(-1); }
 229
 230
 231         fd_delete_nodes = fopen(file_delete_nodes, "w");
 232         if (fd_delete_nodes == NULL) { perror("Open:"); exit(-1); }
 233         fd_delete_ways = fopen(file_delete_ways, "w");
 234         if (fd_delete_ways == NULL) { perror("Open:"); exit(-1); }
 235         fd_delete_relations = fopen(file_delete_relations, "w");
 236         if (fd_delete_relations == NULL) { perror("Open:"); exit(-1); }
 237
 238         fd_modify_nodes = fopen(file_modify_nodes, "w");
 239         if (fd_modify_nodes == NULL) { perror("Open:"); exit(-1); }
 240         fd_modify_ways = fopen(file_modify_ways, "w");
 241         if (fd_modify_ways == NULL) { perror("Open:"); exit(-1); }
 242         fd_modify_relations = fopen(file_modify_relations, "w");
 243         if (fd_modify_relations == NULL) { perror("Open:"); exit(-1); }
 244
 245
 246     nextline;
 247
 248     do {
 249     end = strchrnul((const char*) start, '\n');
 250
 251     nodename = strchrnul(start, '<') + 1;
 252     nodename_fast_end = strchrnul(nodename, '>');
 253     nodename_end = strchrnul(nodename, ' ');
 254
 255     if (nodename_fast_end < nodename_end) nodename_end = nodename_fast_end;
 256
 257     if (nodename[0] == '/') {
 258         free(attr_id);
 259         free(attr_lat);
 260         free(attr_lon);
 261         free(attr_timestamp);
 262         free(attr_changeset);
 263         free(attr_user);
 264         free(attr_uid);
 265         free(attr_version);
 266
 267         attr_id = attr_lat = attr_lon = attr_user = attr_uid = attr_timestamp = attr_changeset = attr_version = NULL;
 268
 269         sequence = 0;
 270
 271         nextline;
 272         continue;
 273     }
 274
 275     switch (nodename_end - nodename) {
 276         case 2:
 277             current_tag = ND;
 278             break;
 279         case 3: {
 280                     switch (nodename[0]) {
 281                         case 'o':
 282                             current_tag = OSM;
 283                             break;
 284                         case 'w':
 285                             current_tag = WAY;
 286                             break;
 287                         case 't':
 288                             current_tag = TAG;
 289                             break;
 290                         default:
 291                             fprintf(stderr, "--> %c%c", nodename[0], nodename[1]);
 292                     }
 293                     break;
 294                 }
 295         case 4:
 296                 current_tag = NODE;
 297                 break;
 298         case 5:
 299                 /* BOUND */
 300                 nextline;
 301                 continue;
 302         case 6:
 303                 switch (nodename[2]) {
 304                         case 'e':
 305                                 ywk = CREATE;
 306                                 nextline;
 307                                 continue;
 308                                 break;
 309                         case 'd':
 310                                 ywk = MODIFY;
 311                                 nextline;
 312                                 continue;
 313                                 break;
 314                         case 'l':
 315                                 ywk = DELETE;
 316                                 nextline;
 317                                 continue;
 318                                 break;
 319                         case 'm':
 320                                 current_tag = MEMBER;
 321                                 break;
 322                         default:
 323                                 fprintf(stderr, "--> %c%c", nodename[0], nodename[1]);
 324                 }
 325
 326                 break;
 327         case 8:
 328                 current_tag = RELATION;
 329                 break;
 330         default:
 331                 fprintf(stderr, "--> %c%c", nodename[0], nodename[1]);
 332     }
 333
 334
 335     char *key, *key_end, *value_end;
 336     key = nodename_end + 1;
 337
 338     do {
 339         char *value;
 340         key_state_t current_key = UNKNOWN;
 341         key_end = strchrnul(key, '=');
 342
 343         if (key_end == NULL || key_end >= end)
 344                 break;
 345
 346         switch (key_end - key) {
 347             case 1: {
 348                     switch (key[0]) {
 349                         case 'k':
 350                             current_key = KEY;
 351                             break;
 352                         case 'v':
 353                             current_key = VALUE;
 354                             break;
 355                         default:
 356                             current_key = UNKNOWN;
 357                     }
 358                     break;
 359             }
 360             case 2:
 361                 current_key = ID;
 362                 break;
 363             case 3: {
 364                         switch (key[1]) {
 365                             case 'a':
 366                                 current_key = LAT;
 367                                 break;
 368                             case 'o':
 369                                 current_key = LON;
 370                                 break;
 371                             case 'e':
 372                                 current_key = REF;
 373                                 break;
 374                             case 'i':
 375                                 current_key = UID;
 376                                 break;
 377                             default:
 378                                 current_key = UNKNOWN;
 379                                 fprintf(stderr, "--> %c%c\n", key[0], key[1]);
 380                         }
 381                         break;
 382                     }
 383             case 4: {
 384                         switch (key[0]) {
 385                             case 'u':
 386                                 current_key = USER;
 387                                 break;
 388                             case 'r':
 389                                 current_key = ROLE;
 390                                 break;
 391                             case 't':
 392                                 current_key = TYPE;
 393                                 break;
 394                             default:
 395                                 current_key = UNKNOWN;
 396                                 fprintf(stderr, "--> %c%c\n", key[0], key[1]);
 397                         }
 398                         break;
 399                 }
 400             case 7: {
 401                 current_key = VERSION;
 402                 break;
 403             }
 404             case 9: {
 405                         switch (key[0]) {
 406                                 case 't':
 407                                         current_key = TIMESTAMP;
 408                                         break;
 409                                 case 'c':
 410                                         current_key = CHANGESET;
 411                                         break;
 412                                 default:
 413                                         current_key = UNKNOWN;
 414                                         fprintf(stderr, "--> %c%c\n", key[0], key[1]);
 415                         }
 416                     break;
 417             }
 418             default: {
 419                     char *thingie = strndup(key, (key_end - key));
 420                     current_key = UNKNOWN;
 421
 422                     fprintf(stderr, "UNKNOWN ATTR %s-> %c%c\n", thingie, key[0], key[1]);
 423                     free(thingie);
 424                 }
 425         }
 426
 427         value = key_end + 2;
 428         value_end = value;
 429         value_end = strchr(value_end, '"');
 430
 431         if (value_end > end)
 432                 break;
 433
 434         switch (current_key) {
 435             case ID:
 436                 if (attr_id) free(attr_id);
 437                 attr_id = strndup(value, (value_end - value));
 438                 break;
 439
 440             case LAT:
 441                 if (attr_lat) free(attr_lat);
 442                 attr_lat = strndup(value, (value_end - value));
 443                 attr_lat_uint = coordtouint(attr_lat);
 444                 break;
 445
 446             case LON:
 447                 if (attr_lon) free(attr_lon);
 448                 attr_lon = strndup(value, (value_end - value));
 449                 attr_lon_uint = coordtouint(attr_lon);
 450                 break;
 451
 452             case TIMESTAMP:
 453                 if (attr_timestamp) free(attr_timestamp);
 454 //              attr_timestamp = strndup(value, (value_end - value));
 455                 attr_timestamp = strndup(value, (value_end - (value + 1))); /* another stupid fix */
 456 //              attr_timestamp[10] = ' '; /* Stupid timestamp fix */
 457                 break;
 458
 459             case CHANGESET:
 460                 if (attr_changeset) free(attr_changeset);
 461                 attr_changeset = strndup(value, (value_end - value));
 462                 break;
 463
 464             case USER: {
 465                 char *tmp;
 466                 if (attr_user) free(attr_user);
 467                 attr_user = strndup(value, (value_end - value));
 468                 tmp = escape_string(attr_user);
 469                 free(attr_user);
 470                 attr_user = tmp;
 471                 break;
 472             }
 473
 474             case UID: {
 475                 if (attr_uid) free(attr_uid);
 476                 attr_uid = strndup(value, (value_end - value));
 477                 break;
 478             }
 479
 480             case VERSION: {
 481                 if (attr_version) free(attr_version);
 482                 attr_version = strndup(value, (value_end - value));
 483                 break;
 484             }
 485
 486             case KEY: {
 487                 char *tmp;
 488                 if (attr_key) free(attr_key);
 489                 attr_key = strndup(value, (value_end - value));
 490                 tmp = escape_string(attr_key);
 491                 free(attr_key);
 492                 attr_key = tmp;
 493                 break;
 494             }
 495
 496             case VALUE: {
 497                 char *tmp;
 498                 if (attr_value) free(attr_value);
 499                 attr_value = strndup(value, (value_end - value));
 500                 tmp = escape_string(attr_value);
 501                 free(attr_value);
 502                 attr_value = tmp;
 503                 break;
 504             }
 505
 506             case TYPE:
 507                 if (attr_type) free(attr_type);
 508                 attr_type = strndup(value, (value_end - value));
 509                 break;
 510
 511             case REF:
 512                 if (attr_ref) free(attr_ref);
 513                 attr_ref = strndup(value, (value_end - value));
 514                 break;
 515
 516             case ROLE: {
 517                 char *tmp;
 518                 if (attr_role) free(attr_role);
 519                 attr_role = strndup(value, (value_end - value));
 520                 tmp = escape_string(attr_role);
 521                 free(attr_role);
 522                 attr_role = tmp;
 523                 break;
 524             }
 525
 526             default:
 527                 fprintf(stderr, "--> %c%c\n", value[0], value[1]);
 528         }
 529
 530         key = value_end + 2;
 531     } while (key < end);
 532
 533     if (ywk == DELETE || ywk == MODIFY) {
 534         FILE *this = NULL;
 535         switch (current_tag) {
 536                 case NODE:
 537                         if (ywk == DELETE) { this = fd_delete_nodes; count_delete_nodes++; } else { this = fd_modify_nodes; count_modify_nodes++; }
 538                         break;
 539                 case WAY:
 540                         if (ywk == DELETE) { this = fd_delete_ways; count_delete_ways++; } else { this = fd_modify_ways; count_modify_ways++; }
 541                         break;
 542                 case RELATION:
 543                         if (ywk == DELETE) { this = fd_delete_relations; count_delete_relations++; } else { this = fd_modify_relations; count_modify_relations++; }
 544                         break;
 545           }
 546           if (this) {
 547                   fputs(attr_id, this);
 548                   fputs("\n", this);
 549           }
 550     }
 551
 552     if (ywk != DELETE) {
 553     switch (current_tag) {
 554         case NODE:
 555             fprintf(fd_nodes, "%s, %s, %s, %s, '%s', %lu\n", attr_id, attr_lon, attr_lat, (attr_uid != NULL ? attr_uid : "0"), attr_timestamp, zcurve(attr_lon, attr_lat));
 556 #ifdef BENCHMARK
 557             fprintf(fd_nodes_uint, "%s, %d, %d, %s, '%s'\n", attr_id, attr_lon_uint, attr_lat_uint, (attr_uid != NULL ? attr_uid : "0"), attr_timestamp);
 558             fprintf(fd_nodes_gis, "%s, 'POINT( %s %s )', '%s', '%s'\n", attr_id, attr_lat, attr_lon, (attr_uid != NULL ? attr_uid : "0"), attr_timestamp);
 559 #endif
 560             count_nodes++;
 561             break;
 562         case TAG: {
 563                 switch (parent_tag) {
 564                         case NODE:
 565                             fprintf(fd_node_tags, "%s, '%s', '%s'\n", attr_id, attr_key, attr_value);
 566                             count_node_tags++;
 567                             break;
 568                         case WAY:
 569                             fprintf(fd_way_tags, "%s, '%s', '%s'\n", attr_id, attr_key, attr_value);
 570                             count_way_tags++;
 571                             break;
 572                         case RELATION:
 573                             fprintf(fd_relation_tags, "%s, '%s', '%s'\n", attr_id, attr_key, attr_value);
 574                             count_relation_tags++;
 575                             break;
 576                         default:
 577                             break;
 578                 }
 579                 break;
 580         }
 581         case WAY:
 582             fprintf(fd_ways, "%s, %s, '%s'\n", attr_id, (attr_uid != NULL ? attr_uid : "0"), attr_timestamp);
 583             count_ways++;
 584 //          fprintf(fd_way_tags, "%s, '%s', '%s'\n", attr_id, "type", "way");
 585 //          count_way_tags++;
 586             break;
 587         case RELATION:
 588             fprintf(fd_relations, "%s, %s, '%s'\n", attr_id, (attr_uid != NULL ? attr_uid : "0"), attr_timestamp);
 589             count_relations++;
 590             break;
 591         case MEMBER:
 592                 if (strcmp(attr_type, "node") == 0) {
 593                         fprintf(fd_members_node, "%s, %lu, %s, '%s'\n", attr_id, sequence, attr_ref, attr_role);
 594                         count_members_node++;
 595                 } else if (strcmp(attr_type, "way") == 0) {
 596                         fprintf(fd_members_way, "%s, %lu, %s, '%s'\n", attr_id, sequence, attr_ref, attr_role);
 597                         count_members_way++;
 598                 } else if (strcmp(attr_type, "relation") == 0) {
 599                         fprintf(fd_members_relation, "%s, %lu, %s, '%s'\n", attr_id, sequence, attr_ref, attr_role);
 600                         count_members_relation++;
 601                 }
 602                 sequence++;
 603                 break;
 604         case ND:
 605             fprintf(fd_way_nds, "%s, %lu, %s\n", attr_id, sequence, attr_ref);
 606             sequence++;
 607             count_way_nds++;
 608             break;
 609         default:
 610             break;
 611     }
 612     }
 613
 614     if (end[-2] == '/') {
 615         switch (current_tag) {
 616             case NODE:
 617                 free(attr_lat);
 618                 free(attr_lon);
 619                 attr_lat = NULL;
 620                 attr_lon = NULL;
 621                 attr_lat_uint = 0;
 622                 attr_lon_uint = 0;
 623                 /* no break! */
 624
 625             case WAY:
 626             case RELATION:
 627                 free(attr_id);
 628                 free(attr_timestamp);
 629                 free(attr_changeset);
 630                 free(attr_user);
 631                 free(attr_uid);
 632                 free(attr_version);
 633
 634                 attr_id = attr_user = attr_uid = attr_timestamp = attr_changeset = attr_version = NULL;
 635
 636                 sequence = 0;
 637                 break;
 638
 639             case TAG:
 640                 free(attr_key);
 641                 free(attr_value);
 642
 643                 attr_key = NULL;
 644                 attr_value = NULL;
 645                 break;
 646
 647             case ND:
 648             case MEMBER:
 649                 free(attr_type);
 650                 free(attr_ref);
 651                 free(attr_role);
 652
 653                 attr_type = NULL;
 654                 attr_ref = NULL;
 655                 attr_role = NULL;
 656             default:
 657                 break;
 658         }
 659     } else if (current_tag == NODE || current_tag == WAY || current_tag == RELATION) {
 660                 parent_tag = current_tag;
 661         }
 662
 663
 664      nextline;
 665     } while (stopcondition);
 666 exit:
 667
 668     free(attr_id);
 669     free(attr_lat);
 670     free(attr_lon);
 671     free(attr_timestamp);
 672     free(attr_changeset);
 673     free(attr_user);
 674     free(attr_uid);
 675     free(attr_version);
 676
 677     free(attr_key);
 678     free(attr_value);
 679
 680     fclose(fd_nodes);
 681 #ifdef BENCHMARK
 682     fclose(fd_nodes_uint);
 683     fclose(fd_nodes_gis);
 684 #endif
 685     fclose(fd_node_tags);
 686     fclose(fd_ways);
 687     fclose(fd_way_tags);
 688     fclose(fd_way_nds);
 689     fclose(fd_relations);
 690     fclose(fd_relation_tags);
 691     fclose(fd_members_node);
 692     fclose(fd_members_relation);
 693     fclose(fd_members_way);
 694
 695     fclose(fd_delete_nodes);
 696     fclose(fd_delete_ways);
 697     fclose(fd_delete_relations);
 698     fclose(fd_modify_nodes);
 699     fclose(fd_modify_ways);
 700     fclose(fd_modify_relations);
 701
 702
 703
 704     char *current = get_current_dir_name();
 705
 706     puts("START TRANSACTION;");
 707
 708     if (ywk == NONE) {
 709     puts("CREATE TABLE nodes_legacy (id integer, long double, lat double, uid integer, timestamp timestamptz, zcurve bigint);");
 710 #ifdef BENCHMARK
 711     puts("CREATE TABLE nodes_legacy_uint (id integer, long integer, lat integer, uid integer, timestamp timestamptz);");
 712     puts("CREATE TABLE nodes_legacy_gis (id integer, poi point, uid integer, timestamp timestamptz);");
 713 #endif
 714     puts("CREATE TABLE node_tags (node integer, k varchar(255), v varchar(1024));");
 715     puts("CREATE TABLE ways (id integer,uid integer, timestamp timestamptz);");
 716     puts("CREATE TABLE way_tags (way integer, k varchar(255), v varchar(1024));");
 717     puts("CREATE TABLE way_nds (way integer, idx integer, to_node integer);");
 718     puts("CREATE TABLE relations(id integer, uid integer, timestamp timestamptz);");
 719     puts("CREATE TABLE relation_members_node (relation integer, idx integer, to_node integer, role varchar(255));");
 720     puts("CREATE TABLE relation_members_relation (relation integer, idx integer, to_relation integer, role varchar(255));");
 721     puts("CREATE TABLE relation_members_way (relation integer, idx integer, to_way integer, role varchar(255));");
 722     puts("CREATE TABLE relation_tags (relation integer, k varchar(255), v varchar(1024));");
 723     } else {
 724       if (count_delete_nodes > 0 || count_delete_ways > 0 || count_delete_relations > 0 ||
 725           count_modify_nodes > 0 || count_modify_ways > 0 || count_modify_relations > 0) {
 726         if (count_delete_relations > 0 || count_modify_relations > 0) {
 727           puts("CREATE TEMPORARY TABLE delete_relations (id integer);");
 728         }
 729         if (count_delete_ways > 0 || count_modify_ways > 0) {
 730           puts("CREATE TEMPORARY TABLE delete_ways (id integer);");
 731         }
 732         if (count_delete_nodes > 0 || count_modify_nodes > 0) {
 733           puts("CREATE TEMPORARY TABLE delete_nodes (id integer);");
 734         }
 735
 736         if (count_delete_nodes > 0 || count_delete_ways > 0 || count_delete_relations > 0) {
 737         if (count_delete_relations > 0) {
 738           printf("COPY %lu RECORDS INTO delete_relations FROM '%s/" file_delete_relations "' USING DELIMITERS ',', '\\n', '''';\n", count_delete_relations, current);
 739         }
 740
 741         if (count_delete_ways > 0) {
 742           printf("COPY %lu RECORDS INTO delete_ways FROM '%s/" file_delete_ways "' USING DELIMITERS ',', '\\n', '''';\n", count_delete_ways, current);
 743         }
 744
 745         if (count_delete_nodes > 0) {
 746           printf("COPY %lu RECORDS INTO delete_nodes FROM '%s/" file_delete_nodes "' USING DELIMITERS ',', '\\n', '''';\n", count_delete_nodes, current);
 747         }
 748
 749         if (count_delete_relations > 0) {
 750           puts("DELETE FROM relation_members_relation WHERE to_relation IN (SELECT id FROM delete_relations);");
 751         }
 752
 753         if (count_delete_ways > 0) {
 754           puts("DELETE FROM relation_members_way WHERE to_way IN (SELECT id FROM delete_ways);");
 755         }
 756
 757         if (count_delete_nodes > 0) {
 758           puts("DELETE FROM relation_members_node WHERE to_node IN (SELECT id FROM delete_nodes);\n"
 759                "DELETE FROM way_nds WHERE to_node IN (SELECT id FROM delete_nodes);");
 760         }
 761         }
 762
 763         if (count_modify_relations > 0) {
 764           printf("COPY %lu RECORDS INTO delete_relations FROM '%s/" file_modify_relations "' USING DELIMITERS ',', '\\n', '''';\n", count_modify_relations, current);
 765         }
 766
 767         if (count_modify_ways > 0) {
 768           printf("COPY %lu RECORDS INTO delete_ways FROM '%s/" file_modify_ways "' USING DELIMITERS ',', '\\n', '''';\n", count_modify_ways, current);
 769         }
 770
 771         if (count_modify_nodes > 0) {
 772           printf("COPY %lu RECORDS INTO delete_nodes FROM '%s/" file_modify_nodes "' USING DELIMITERS ',', '\\n', '''';\n", count_modify_nodes, current);
 773         }
 774
 775         if (count_delete_relations > 0 || count_modify_relations > 0) {
 776           puts("DELETE FROM relation_members_relation WHERE relation IN (SELECT id FROM delete_relations);\n"
 777                "DELETE FROM relation_members_way WHERE relation IN (SELECT id FROM delete_relations);\n"
 778                "DELETE FROM relation_members_node WHERE relation IN (SELECT id FROM delete_relations);\n"
 779                "DELETE FROM relation_tags WHERE relation IN (SELECT id FROM delete_relations);\n"
 780                "DELETE FROM relations WHERE id IN (SELECT id FROM delete_relations);\n"
 781                "DROP TABLE delete_relations;\n");
 782         }
 783
 784         if (count_delete_ways > 0 || count_modify_ways > 0) {
 785           puts("DELETE FROM way_nds WHERE way IN (SELECT id FROM delete_ways);\n"
 786                "DELETE FROM way_tags WHERE way IN (SELECT id FROM delete_ways);\n"
 787                "DELETE FROM ways WHERE id IN (SELECT id FROM delete_ways);\n"
 788                "DROP TABLE delete_ways;\n");
 789         }
 790
 791         if (count_delete_nodes > 0 || count_modify_nodes > 0) {
 792           puts("DELETE FROM node_tags WHERE node IN (SELECT id FROM delete_nodes);\n"
 793                "DELETE FROM nodes_legacy WHERE id IN (SELECT id FROM delete_nodes);\n"
 794                "DROP TABLE delete_nodes;\n");
 795         }
 796       }
 797     }
 798
 799     printf("COPY %lu RECORDS INTO nodes_legacy from '%s/" file_nodes "' USING DELIMITERS ',', '\\n', '''';\n", count_nodes, current);
 800 #ifdef BENCHMARK
 801     printf("COPY %lu RECORDS INTO nodes_legacy_uint from '%s/" file_nodes_uint "' USING DELIMITERS ',', '\\n', '''';\n", count_nodes, current);
 802     printf("COPY %lu RECORDS INTO nodes_legacy_gis from '%s/" file_nodes_gis "' USING DELIMITERS ',', '\\n', '''';\n", count_nodes, current);
 803 #endif
 804     printf("COPY %lu RECORDS INTO node_tags from '%s/" file_node_tags "' USING DELIMITERS ',', '\\n', '''';\n", count_node_tags, current);
 805     printf("COPY %lu RECORDS INTO ways from '%s/" file_ways "' USING DELIMITERS ',', '\\n', '''';\n", count_ways, current);
 806     printf("COPY %lu RECORDS INTO way_tags from '%s/" file_way_tags "' USING DELIMITERS ',', '\\n', '''';\n", count_way_tags, current);
 807     printf("COPY %lu RECORDS INTO way_nds from '%s/" file_way_nds "' USING DELIMITERS ',', '\\n', '''';\n", count_way_nds, current);
 808     printf("COPY %lu RECORDS INTO relations from '%s/" file_relations "' USING DELIMITERS ',', '\\n', '''';\n", count_relations, current);
 809     printf("COPY %lu RECORDS INTO relation_tags from '%s/" file_relation_tags "' USING DELIMITERS ',', '\\n', '''';\n", count_relation_tags, current);
 810     printf("COPY %lu RECORDS INTO relation_members_node from '%s/" file_relation_member_node "' USING DELIMITERS ',', '\\n', '''';\n", count_members_node, current);
 811     printf("COPY %lu RECORDS INTO relation_members_relation from '%s/" file_relation_member_relation "' USING DELIMITERS ',', '\\n', '''';\n", count_members_relation, current);
 812     printf("COPY %lu RECORDS INTO relation_members_way from '%s/" file_relation_member_way "' USING DELIMITERS ',', '\\n', '''';\n", count_members_way, current);
 813
 814     free(current);
 815
 816     puts("COMMIT;");
 817
 818     if (ywk == NONE) {
 819     puts("START TRANSACTION;");
 820
 821     puts("CREATE SEQUENCE s_nodes AS INTEGER;");
 822     puts("ALTER SEQUENCE s_nodes RESTART WITH (SELECT MAX(id) FROM nodes_legacy);");
 823     puts("ALTER TABLE nodes_legacy ALTER COLUMN id SET NOT NULL;");
 824     puts("ALTER TABLE nodes_legacy ALTER COLUMN id SET DEFAULT NEXT VALUE FOR \"sys\".\"s_nodes\";");
 825     puts("ALTER TABLE nodes_legacy ADD CONSTRAINT pk_nodes_id PRIMARY KEY (id);");
 826
 827     puts("CREATE SEQUENCE s_ways AS INTEGER;");
 828     puts("ALTER SEQUENCE s_ways RESTART WITH (SELECT MAX(id) FROM ways);");
 829     puts("ALTER TABLE ways ALTER COLUMN id SET NOT NULL;");
 830     puts("ALTER TABLE ways ALTER COLUMN id SET DEFAULT NEXT VALUE FOR \"sys\".\"s_ways\";");
 831     puts("ALTER TABLE ways ADD CONSTRAINT pk_ways_id PRIMARY KEY (id);");
 832
 833     puts("CREATE SEQUENCE s_relations AS INTEGER;");
 834     puts("ALTER SEQUENCE s_relations RESTART WITH (SELECT MAX(id) FROM relations);");
 835     puts("ALTER TABLE relations ALTER COLUMN id SET NOT NULL;");
 836     puts("ALTER TABLE relations ALTER COLUMN id SET DEFAULT NEXT VALUE FOR \"sys\".\"s_relations\";");
 837     puts("ALTER TABLE relations ADD CONSTRAINT pk_relations_id PRIMARY KEY (id);");
 838
 839     puts("ALTER TABLE relation_members_node ADD CONSTRAINT pk_relation_members_node PRIMARY KEY (relation, idx);");
 840     puts("ALTER TABLE relation_members_way ADD CONSTRAINT pk_relation_members_way PRIMARY KEY (relation,idx);");
 841     puts("ALTER TABLE relation_members_relation ADD CONSTRAINT pk_relation_members_relation PRIMARY KEY (relation,idx);");
 842
 843     puts("COMMIT;");
 844
 845
 846     puts("START TRANSACTION;");
 847
 848     puts("ALTER TABLE node_tags ADD CONSTRAINT fk_node_tags_node FOREIGN KEY (node) REFERENCES nodes_legacy (id);");
 849     puts("ALTER TABLE node_tags ADD CONSTRAINT pk_node_tags UNIQUE (node, k, v);");
 850
 851     puts("ALTER TABLE way_tags ADD CONSTRAINT fk_way_tags_way FOREIGN KEY (way) REFERENCES ways (id);");
 852     puts("ALTER TABLE way_tags ADD CONSTRAINT pk_way_tags UNIQUE (way, k, v);");
 853
 854     puts("ALTER TABLE way_nds ADD CONSTRAINT pk_way_nds PRIMARY KEY (way, idx);");
 855     puts("ALTER TABLE way_nds ADD CONSTRAINT fk_way_nds_way FOREIGN KEY (way) REFERENCES ways (id);");
 856     puts("ALTER TABLE way_nds ADD CONSTRAINT fk_way_nds_node FOREIGN KEY (to_node) REFERENCES nodes_legacy (id);");
 857
 858     puts("ALTER TABLE relation_tags ADD CONSTRAINT fk_relation_tags FOREIGN KEY (relation) REFERENCES relations (id);");
 859     puts("ALTER TABLE relation_tags ADD CONSTRAINT pk_relation_tags UNIQUE (relation, k, v);");
 860
 861     puts("ALTER TABLE relation_members_node ADD CONSTRAINT fk_relation_members_node FOREIGN KEY (relation) REFERENCES relations (id);");
 862     puts("ALTER TABLE relation_members_node ADD CONSTRAINT fk_relation_members_tonode FOREIGN KEY (to_node) REFERENCES nodes_legacy (id);");
 863
 864     puts("ALTER TABLE relation_members_way ADD CONSTRAINT fk_relation_members_way FOREIGN KEY (relation) REFERENCES relations (id);");
 865     puts("ALTER TABLE relation_members_way ADD CONSTRAINT fk_relation_members_toway FOREIGN KEY (to_way) REFERENCES ways (id);");
 866
 867     puts("ALTER TABLE relation_members_relation ADD CONSTRAINT fk_relation_members_relation FOREIGN KEY (relation) REFERENCES relations (id);");
 868     puts("ALTER TABLE relation_members_relation ADD CONSTRAINT fk_relation_members_torelation FOREIGN KEY (to_relation) REFERENCES relations (id);");
 869
 870     puts("COMMIT;");
 871     }
 872 }
 873
 874
 875 int main(int argc, char *argv[]) {
 876 #ifdef MMAP
 877     int fd;
 878     struct stat statbuf;
 879
 880     if (argc != 2)
 881         exit(-1);
 882
 883     fprintf(stderr, "Analysing %s...\n", argv[1]);
 884
 885     fd = open(argv[1], O_RDONLY);
 886
 887     if (fd < 0)
 888         exit(-1);
 889
 890     if (fstat (fd, &statbuf) == -1) { perror("fstat:"); exit(-1); }
 891
 892     if (statbuf.st_size > 0) {
 893         char *range = NULL;
 894         range = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED, fd, (off_t) 0);
 895         if (range == MAP_FAILED) { perror("Mmap:"); puts("(did you compile PAE in the kernel?)"); exit(-1); }
 896         parser(range, statbuf.st_size / sizeof(char));
 897         munmap(range, statbuf.st_size);
 898     }
 899
 900     close(fd);
 901 #else
 902     parser();
 903 #endif
 904     exit(0);
 905 }