/* javac -d ./class parser/*.java java -Dfile.encoding=UTF-8 -Xmx5g -classpath ./class Parser -enc japan java -Dfile.encoding=UTF-8 -Xmx5g -classpath ./class Parser -rel japan java -Dfile.encoding=UTF-8 -Xmx5g -classpath ./class Parser -run hot */ import java.io.*; import java.nio.*; import java.util.*; import java.nio.file.*; public class Parser { String srcDir; String dstDir = "c:/map/dat/"; HashMap mapWays; DataOutputStream dos; void parseWay(String name) { srcDir = "c:/map/obf/" + name + "/"; dos = getDataOutputStream(name); int cnt = 0, cntAll = 0, cntTag = 0, cntNode = 0, cntInt = 0; for (int ix = 0; ; ix++) { String src = srcDir + "way" + ix + ".obf"; if (!Files.exists(Paths.get(src))) { break; } try (DataInputStream dis = new DataInputStream( new BufferedInputStream(new FileInputStream(src)))) { while (true) { long wid = dis.readLong(); MyTag[] tags = readTags(dis); short num_nodes = dis.readShort(); int[] lons = new int[num_nodes]; int[] lats = new int[num_nodes]; for (int i = 0; i < num_nodes; i++) { lons[i] = dis.readInt(); lats[i] = dis.readInt(); } boolean diff = true; for (int i = 1; i < num_nodes; i++) { int diffLon = lons[i] - lons[i-1]; int diffLat = lats[i] - lats[i-1]; if (diffLon > Short.MAX_VALUE || diffLat > Short.MAX_VALUE || diffLon < Short.MIN_VALUE || diffLat < Short.MIN_VALUE) { diff = false; //System.out.println((++cntInt) + ": " + diffLon + ", " + diffLat); break; // 差分を2バイトで表せない } } cntAll++; boolean isBuilding = false; for (MyTag tag : tags) { if (tag.key == Key.building) { isBuilding = true; break; } } if (isBuilding && lons[0]==lons[num_nodes-1] && lats[0]==lats[num_nodes-1]) { cnt++; cntTag += tags.length; cntNode += num_nodes; if (!diff) { dos.writeShort(tags.length + 1); // num_tags + 1 writeTags(dos, tags); dos.writeShort((short)Key.int_code.ordinal()); dos.writeShort(num_nodes); for (int i = 0; i < num_nodes; i++) { dos.writeInt(lons[i]); dos.writeInt(lats[i]); } } else { // default dos.writeShort(tags.length); // num_tags writeTags(dos, tags); dos.writeShort(num_nodes); dos.writeInt(lons[0]); dos.writeInt(lats[0]); for (int i = 1; i < num_nodes; i++) { dos.writeShort((short)(lons[i]-lons[i-1])); dos.writeShort((short)(lats[i]-lats[i-1])); } } } } } catch (EOFException e) { ; } catch (Exception e) { e.printStackTrace(); } } System.out.println("建物レコード数=" + cnt + ", Wayレコード=" + cntAll + ", 建物レコードの平均タグ数=" + (float)cntTag/cnt + ", 建物レコードの平均Node数=" + (float)cntNode/cnt ); close(dos); } public static class MyTag { Key key; Val val; // codeタグのときの値 String keyval; String sval; // 文字列タグのときの値 int ival; // intタグのときの値 float fval; // floatタグのときの値 public MyTag() { keyval = null; } } void writeTags(DataOutputStream dos, MyTag[] tags) { try { // dos.writeShort(tags.length); // num_tags for (MyTag tag : tags) { short ikey = (short)tag.key.ordinal(); dos.writeShort(ikey); // key if (ikey < Key.endCodeKey.ordinal() || ikey > Key.endFloatKey.ordinal()) { dos.writeShort((short)tag.val.ordinal()); // val } else if (ikey < Key.endStrKey.ordinal()) { byte[] bytes = tag.sval.getBytes(); dos.writeShort((short)bytes.length); dos.write(bytes); } else if (ikey < Key.endIntKey.ordinal()) { dos.writeInt(tag.ival); } else if (ikey <= Key.endFloatKey.ordinal()) { dos.writeFloat(tag.fval); } else { System.out.printf("Error key=%s[%d]\n", Tag.keys[ikey].name(), ikey); continue; } } } catch (Exception e) { e.printStackTrace(); } } MyTag[] readTags(DataInputStream dis) { List listMyTags = new ArrayList<>(); try { short num_tags = dis.readShort(); while (num_tags-- > 0) { MyTag tag = new MyTag(); short ikey = dis.readShort(); tag.key = Tag.keys[ikey]; if (ikey < Key.endCodeKey.ordinal() || ikey > Key.endFloatKey.ordinal()) { short ival = dis.readShort(); tag.val = Tag.vals[ival]; tag.keyval = tag.key.name() + "=" + tag.val.name(); //if (tag.key == Key.building) tag.building = tag.val; } else if (ikey < Key.endStrKey.ordinal()) { short len = dis.readShort(); if (len < 0) System.out.printf("Error key=%s[%d]\n", Tag.keys[ikey].name(), ikey); byte[] buff = new byte[len]; dis.read(buff); tag.sval = new String(buff); } else if (ikey < Key.endIntKey.ordinal()) { tag.ival = dis.readInt(); } else if (ikey <= Key.endFloatKey.ordinal()) { tag.fval = dis.readFloat(); } else { System.out.printf("Error key=%s[%d]\n", Tag.keys[ikey].name(), ikey); continue; } listMyTags.add(tag); } } catch (EOFException e) { ; } catch (Exception e) { e.printStackTrace(); } return listMyTags.toArray(new MyTag[0]); } public static class Way { long id; MyTag[] tags; long[] lonlats; // lon<<32 + lat List rids; long rid; // 現在処理中の Relation ID boolean reversed; public Way(long id, MyTag[] tags, long[] lonlats) { this.id = id; this.tags = tags; this.lonlats = lonlats; rids = new ArrayList<>(); reversed = false; } } DataOutputStream getDataOutputStream(String name) { try { String path = dstDir + name + ".dat"; return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path))); } catch (IOException ex) { ex.printStackTrace(); return null; } } void close(DataOutputStream dos) { try { dos.flush(); dos.close(); //totalSizeFile += sizeFile; } catch (IOException ex) { ex.printStackTrace(); } } public static void main(String[] args) throws Exception { if ("-enc".equals(args[0])) { Encoder.run(args[1]); } else if ("-run".equals(args[0])) { Parser parser = new Parser(); parser.parseWay(args[1]); } else if ("-rel".equals(args[0])) { Relation.run(args[1]); } else if ("-check".equals(args[0])) { Checker.check(args[1]); } } } /**************************************************************** public class Encoder extends DefaultHandler { enum Section { none, node, way, relation } final double E7 = 10000000.0; String dirDst; DataOutputStream dos; int cntFile; int sizeFile; // 単位: バイト Section section = Section.none; static long totalSizeFile = 0; static long maxId = 0; static int nodes=0, ways=0, rels=0; int numTags; // オブジェクト毎のタグ数 int numMembers; ByteBuffer bbTags; ByteBuffer bbNodes; ByteBuffer bbMembers; HashSet setNodes; // 有効タグを持つノード(ファイル出力したもの) long id; int lon, lat; long[] nids = new long[0x10000000]; int[] lons = new int[0x10000000]; int[] lats = new int[0x10000000]; int nEntry; public Encoder(String dir) { this.dirDst = dir; nEntry = 0; bbTags = ByteBuffer.allocate(10 * 1000); bbNodes = ByteBuffer.allocate(1000 * 1000); bbMembers = ByteBuffer.allocate(1000 * 1000); } DataOutputStream getDataOutputStream() { try { sizeFile = 0; String path = dirDst + section.name() + cntFile + ".obf"; return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path))); } catch (IOException ex) { ex.printStackTrace(); return null; } } void close(DataOutputStream dos) { try { dos.flush(); dos.close(); totalSizeFile += sizeFile; } catch (IOException ex) { ex.printStackTrace(); } } int getLonLatIx(long id) { int ix = binary_search(id, 0, nEntry-1); if (ix < 0) { System.out.println("座標値が見つからない id=" + id); } return ix; } int binary_search(long id, int imin, int imax) { if (imax < imin) { return -1; // id が見つからなかった } else { int imid = imin + (imax - imin) / 2; if (nids[imid] > id) { return binary_search(id, imin, imid - 1); } else if (nids[imid] < id) { return binary_search(id, imid + 1, imax); } else { return imid; } } } void putString(ByteBuffer bb, String str) { byte[] ba = str.getBytes(); bb.putShort((short)ba.length); // バイト単位の長さ bb.put(ba); } public void startElement (String namespaceURI, String localName, String qName, Attributes atts ) { if (qName.equals("node")) { // nodeオブジェクトの開始処理 nodes++; numTags = 0; bbTags.clear(); // = ByteBuffer.allocate(10 * 1000); if (section != Section.node) { System.out.println("Node Section starts."); section = Section.node; cntFile = 0; dos = getDataOutputStream(); setNodes = new HashSet<>(); } String s_id = atts.getValue("id"); String s_lon = atts.getValue("lon"); String s_lat = atts.getValue("lat"); try { id = Long.parseLong(s_id); lon = (int)(Double.parseDouble(s_lon) * E7); lat = (int)(Double.parseDouble(s_lat) * E7); nids[nEntry] = id; lons[nEntry] = lon; lats[nEntry] = lat; nEntry++; if (id > maxId) maxId = id; } catch (NumberFormatException e) { System.out.printf("Error node: %s %s %s\n", s_id, s_lon, s_lat); } } else if (qName.equals("way")) { // wayオブジェクトの開始処理 ways++; numTags = 0; bbTags.clear(); // = ByteBuffer.allocate(10 * 1000); bbNodes.clear(); // = ByteBuffer.allocate(1000 * 1000); if (section != Section.way) { System.out.println("Way Section starts."); section = Section.way; close(dos); cntFile = 0; dos = getDataOutputStream(); } String s_id = atts.getValue("id"); try { id = Long.parseLong(s_id); } catch (NumberFormatException e) { System.out.printf("Error way: %s\n", s_id); } } else if (qName.equals("relation")) { // relationオブジェクトの開始処理 rels++; numTags = 0; bbTags.clear(); // = ByteBuffer.allocate(10*1000); numMembers = 0; bbMembers.clear(); // = ByteBuffer.allocate(1000 * 1000); if (section != Section.relation) { System.out.println("Relation Section starts."); section = Section.relation; cntFile = 0; close(dos); dos = getDataOutputStream(); } String s_id = atts.getValue("id"); try { id = Long.parseLong(s_id); } catch (NumberFormatException e) { System.out.printf("Error relation: %s\n", s_id); } } else if (qName.equals("tag")) { // tagオブジェクトの開始処理 … node, way, relationの子供 String skey = atts.getValue("k"); if (Tag.setKeys.contains(skey)) { String val = atts.getValue("v"); Key key = Key.valueOf(skey); int ikey = key.ordinal(); numTags++; if (ikey < Key.endCodeKey.ordinal() || ikey > Key.endFloatKey.ordinal()) { if (Tag.setVals.contains(val)) { int ival = Val.valueOf(val).ordinal(); bbTags.putInt(ikey); bbTags.putInt(ival); } } else if (ikey < Key.endStrKey.ordinal()) { bbTags.putInt(ikey); putString(bbTags, val); } else if (ikey < Key.endIntKey.ordinal()) { try { int v = Integer.parseInt(val); bbTags.putInt(ikey); bbTags.putInt(v); } catch (NumberFormatException e) { try { Key s_key = Key.valueOf("s_" + skey); bbTags.putInt(s_key.ordinal()); putString(bbTags, val); } catch (Exception ex) { System.out.println("Error IntTag: " + key + "=" + val); } } // エラーがあったときはタグを無視する } else if (ikey <= Key.endFloatKey.ordinal()) { val = val.replace("m", ""); try { float v = Float.parseFloat(val); bbTags.putInt(ikey); bbTags.putFloat(v); } catch (NumberFormatException e) { try { Key s_key = Key.valueOf("s_" + skey); bbTags.putInt(s_key.ordinal()); putString(bbTags, val); } catch (Exception ex) { System.out.println("Error FloatTag: " + key + "=" + val); } } } else { numTags--; // 取り消し System.out.println(key + "=" + val); } } } else if (qName.equals("nd")) { // ndオブジェクトの開始処理 … wayオブジェクトの子供 String ref = atts.getValue("ref"); try { long v = Long.parseLong(ref); int ix = getLonLatIx(v); bbNodes.putInt(lons[ix]); bbNodes.putInt(lats[ix]); } catch (NumberFormatException e) { System.out.println("Error nd.ref: " + ref); } // ここでのエラーはない } else if (qName.equals("member")) { // memberオブジェクトの開始処理 … relationオブジェクトの子供 String s_type = atts.getValue("type"); String s_ref = atts.getValue("ref"); String s_role = atts.getValue("role"); try { long v = Long.parseLong(s_ref); bbMembers.putShort((short)s_type.charAt(0)); // 'n'(node), 'w'(way), 'r'(relation) bbMembers.putLong(v); putString(bbMembers, s_role); numMembers++; } catch (NumberFormatException e) { System.out.println("Error nd.ref: " + s_ref); } // ここでのエラーはない //System.out.printf("type='%d' ref='%s' role='%s'\n", type, s_ref, s_role); } } public void endElement(String namespaceURI, String localName, String qName) { if (qName.equals("node")) { // nodeオブジェクトの終了処理 if (numTags > 0) { writeLong(id); writeByteBuffer(numTags, bbTags); writeInt(lon); writeInt(lat); } } else if (qName.equals("way")) { // wayオブジェクトの終了処理 writeLong(id); writeByteBuffer(numTags, bbTags); writeByteBuffer(bbNodes.position()/8, bbNodes); } else if (qName.equals("relation")) { // relationオブジェクトの終了処理 writeLong(id); writeByteBuffer(numTags, bbTags); writeByteBuffer(numMembers, bbMembers); } else if (qName.equals("osm")) { close(dos); // 最後の出力ファイルをクローズする } if (sizeFile >= 1024*1024*1024) { close(dos); cntFile++; dos = getDataOutputStream(); } } void writeByteBuffer(int numTags, ByteBuffer bbTags) { if (bbTags.position() > 0) { try { dos.writeShort((short)numTags); dos.write(bbTags.array(), 0, bbTags.position()); sizeFile += bbTags.position(); } catch (IOException e) { e.printStackTrace(); } } } void writeInt(int n) { try { dos.writeInt(n); sizeFile += 8; } catch (IOException ex) { ex.printStackTrace(); } } void writeLong(long n) { try { dos.writeLong(n); sizeFile += 8; } catch (IOException ex) { ex.printStackTrace(); } } public static void main( String[] args ) throws Exception { long start = System.currentTimeMillis(); String src = "c:/map/data/" + args[0] + ".osm"; String dstDir = "c:/map/obf/" + args[0] + "/"; SAXParserFactory factory = SAXParserFactory.newInstance(); SAXParser parser = factory.newSAXParser(); Encoder handler = new Encoder(dstDir); parser.parse(Paths.get(src).toFile(), handler); System.out.printf("maxId=%X, nodes=%X, ways=%X, rels=%X, 総サイズ=%.2fGB 実行時間: %.2f分\n", maxId, nodes, ways, rels, totalSizeFile/1024.0/1024/1024, (System.currentTimeMillis()-start)/60000.0); } } ***************************************/