001package org.openstreetmap.gui.jmapviewer; 002 003//License: GPL. Copyright 2008 by Jan Peter Stotz 004 005import java.io.BufferedReader; 006import java.io.ByteArrayInputStream; 007import java.io.ByteArrayOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileNotFoundException; 011import java.io.FileOutputStream; 012import java.io.IOException; 013import java.io.InputStream; 014import java.io.InputStreamReader; 015import java.io.OutputStreamWriter; 016import java.io.PrintWriter; 017import java.net.HttpURLConnection; 018import java.net.URL; 019import java.net.URLConnection; 020import java.nio.charset.Charset; 021import java.util.HashMap; 022import java.util.Map; 023import java.util.Map.Entry; 024import java.util.Random; 025import java.util.logging.Level; 026import java.util.logging.Logger; 027 028import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; 029import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; 030import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 032import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 034import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; 035 036/** 037 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and 038 * saves all loaded files in a directory located in the temporary directory. 039 * If a tile is present in this file cache it will not be loaded from OSM again. 040 * 041 * @author Jan Peter Stotz 042 * @author Stefan Zeller 043 */ 044public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader { 045 046 private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName()); 047 048 private static final String ETAG_FILE_EXT = ".etag"; 049 private static final String TAGS_FILE_EXT = ".tags"; 050 051 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8"); 052 053 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24; 054 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7; 055 056 protected String cacheDirBase; 057 058 protected final Map<TileSource, File> sourceCacheDirMap; 059 060 protected long maxCacheFileAge = FILE_AGE_ONE_WEEK; 061 protected long recheckAfter = FILE_AGE_ONE_DAY; 062 063 public static File getDefaultCacheDir() throws SecurityException { 064 String tempDir = null; 065 String userName = System.getProperty("user.name"); 066 try { 067 tempDir = System.getProperty("java.io.tmpdir"); 068 } catch (SecurityException e) { 069 log.log(Level.WARNING, 070 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " 071 + e.toString()); 072 throw e; // rethrow 073 } 074 try { 075 if (tempDir == null) 076 throw new IOException("No temp directory set"); 077 String subDirName = "JMapViewerTiles"; 078 // On Linux/Unix systems we do not have a per user tmp directory. 079 // Therefore we add the user name for getting a unique dir name. 080 if (userName != null && userName.length() > 0) { 081 subDirName += "_" + userName; 082 } 083 File cacheDir = new File(tempDir, subDirName); 084 return cacheDir; 085 } catch (Exception e) { 086 } 087 return null; 088 } 089 090 /** 091 * Create a OSMFileCacheTileLoader with given cache directory. 092 * If cacheDir is not set or invalid, IOException will be thrown. 093 * @param map the listener checking for tile load events (usually the map for display) 094 * @param cacheDir directory to store cached tiles 095 */ 096 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { 097 super(map); 098 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) 099 throw new IOException("Cannot access cache directory"); 100 101 log.finest("Tile cache directory: " + cacheDir); 102 cacheDirBase = cacheDir.getAbsolutePath(); 103 sourceCacheDirMap = new HashMap<TileSource, File>(); 104 } 105 106 /** 107 * Create a OSMFileCacheTileLoader with system property temp dir. 108 * If not set an IOException will be thrown. 109 * @param map the listener checking for tile load events (usually the map for display) 110 */ 111 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException { 112 this(map, getDefaultCacheDir()); 113 } 114 115 @Override 116 public TileJob createTileLoaderJob(final Tile tile) { 117 return new FileLoadJob(tile); 118 } 119 120 protected File getSourceCacheDir(TileSource source) { 121 File dir = sourceCacheDirMap.get(source); 122 if (dir == null) { 123 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); 124 if (!dir.exists()) { 125 dir.mkdirs(); 126 } 127 } 128 return dir; 129 } 130 131 protected class FileLoadJob implements TileJob { 132 InputStream input = null; 133 134 Tile tile; 135 File tileCacheDir; 136 File tileFile = null; 137 long fileAge = 0; 138 boolean fileTilePainted = false; 139 140 public FileLoadJob(Tile tile) { 141 this.tile = tile; 142 } 143 144 @Override 145 public Tile getTile() { 146 return tile; 147 } 148 149 @Override 150 public void run() { 151 synchronized (tile) { 152 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading()) 153 return; 154 tile.loaded = false; 155 tile.error = false; 156 tile.loading = true; 157 } 158 tileCacheDir = getSourceCacheDir(tile.getSource()); 159 if (loadTileFromFile()) { 160 return; 161 } 162 if (fileTilePainted) { 163 TileJob job = new TileJob() { 164 165 @Override 166 public void run() { 167 loadOrUpdateTile(); 168 } 169 @Override 170 public Tile getTile() { 171 return tile; 172 } 173 }; 174 JobDispatcher.getInstance().addJob(job); 175 } else { 176 loadOrUpdateTile(); 177 } 178 } 179 180 protected void loadOrUpdateTile() { 181 try { 182 URLConnection urlConn = loadTileFromOsm(tile); 183 if (tileFile != null) { 184 switch (tile.getSource().getTileUpdate()) { 185 case IfModifiedSince: 186 urlConn.setIfModifiedSince(fileAge); 187 break; 188 case LastModified: 189 if (!isOsmTileNewer(fileAge)) { 190 log.finest("LastModified test: local version is up to date: " + tile); 191 tile.setLoaded(true); 192 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 193 return; 194 } 195 break; 196 } 197 } 198 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { 199 String fileETag = tile.getValue("etag"); 200 if (fileETag != null) { 201 switch (tile.getSource().getTileUpdate()) { 202 case IfNoneMatch: 203 urlConn.addRequestProperty("If-None-Match", fileETag); 204 break; 205 case ETag: 206 if (hasOsmTileETag(fileETag)) { 207 tile.setLoaded(true); 208 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge 209 + recheckAfter); 210 return; 211 } 212 } 213 } 214 tile.putValue("etag", urlConn.getHeaderField("ETag")); 215 } 216 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { 217 // If we are isModifiedSince or If-None-Match has been set 218 // and the server answers with a HTTP 304 = "Not Modified" 219 log.finest("ETag test: local version is up to date: " + tile); 220 tile.setLoaded(true); 221 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 222 return; 223 } 224 225 loadTileMetadata(tile, urlConn); 226 saveTagsToFile(); 227 228 if ("no-tile".equals(tile.getValue("tile-info"))) 229 { 230 tile.setError("No tile at this zoom level"); 231 listener.tileLoadingFinished(tile, true); 232 } else { 233 for(int i = 0; i < 5; ++i) { 234 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { 235 Thread.sleep(5000+(new Random()).nextInt(5000)); 236 continue; 237 } 238 byte[] buffer = loadTileInBuffer(urlConn); 239 if (buffer != null) { 240 tile.loadImage(new ByteArrayInputStream(buffer)); 241 tile.setLoaded(true); 242 listener.tileLoadingFinished(tile, true); 243 saveTileToFile(buffer); 244 break; 245 } 246 } 247 } 248 } catch (Exception e) { 249 tile.setError(e.getMessage()); 250 listener.tileLoadingFinished(tile, false); 251 if (input == null) { 252 try { 253 System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage()); 254 } catch(IOException i) { 255 } 256 } 257 } finally { 258 tile.loading = false; 259 tile.setLoaded(true); 260 } 261 } 262 263 protected boolean loadTileFromFile() { 264 FileInputStream fin = null; 265 try { 266 tileFile = getTileFile(); 267 if (!tileFile.exists()) 268 return false; 269 270 loadTagsFromFile(); 271 if ("no-tile".equals(tile.getValue("tile-info"))) 272 { 273 tile.setError("No tile at this zoom level"); 274 if (tileFile.exists()) { 275 tileFile.delete(); 276 } 277 tileFile = getTagsFile(); 278 } else { 279 fin = new FileInputStream(tileFile); 280 if (fin.available() == 0) 281 throw new IOException("File empty"); 282 tile.loadImage(fin); 283 fin.close(); 284 } 285 286 fileAge = tileFile.lastModified(); 287 boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge; 288 if (!oldTile) { 289 tile.setLoaded(true); 290 listener.tileLoadingFinished(tile, true); 291 fileTilePainted = true; 292 return true; 293 } 294 listener.tileLoadingFinished(tile, true); 295 fileTilePainted = true; 296 } catch (Exception e) { 297 try { 298 if (fin != null) { 299 fin.close(); 300 tileFile.delete(); 301 } 302 } catch (Exception e1) { 303 } 304 tileFile = null; 305 fileAge = 0; 306 } 307 return false; 308 } 309 310 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException { 311 input = urlConn.getInputStream(); 312 try { 313 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); 314 byte[] buffer = new byte[2048]; 315 boolean finished = false; 316 do { 317 int read = input.read(buffer); 318 if (read >= 0) { 319 bout.write(buffer, 0, read); 320 } else { 321 finished = true; 322 } 323 } while (!finished); 324 if (bout.size() == 0) 325 return null; 326 return bout.toByteArray(); 327 } finally { 328 input.close(); 329 input = null; 330 } 331 } 332 333 /** 334 * Performs a <code>HEAD</code> request for retrieving the 335 * <code>LastModified</code> header value. 336 * 337 * Note: This does only work with servers providing the 338 * <code>LastModified</code> header: 339 * <ul> 340 * <li>{@link tilesources.OsmTileSource.CycleMap} - supported</li> 341 * <li>{@link tilesources.OsmTileSource.Mapnik} - not supported</li> 342 * </ul> 343 * 344 * @param fileAge time of the 345 * @return <code>true</code> if the tile on the server is newer than the 346 * file 347 * @throws IOException 348 */ 349 protected boolean isOsmTileNewer(long fileAge) throws IOException { 350 URL url; 351 url = new URL(tile.getUrl()); 352 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 353 prepareHttpUrlConnection(urlConn); 354 urlConn.setRequestMethod("HEAD"); 355 urlConn.setReadTimeout(30000); // 30 seconds read timeout 356 // System.out.println("Tile age: " + new 357 // Date(urlConn.getLastModified()) + " / " 358 // + new Date(fileAge)); 359 long lastModified = urlConn.getLastModified(); 360 if (lastModified == 0) 361 return true; // no LastModified time returned 362 return (lastModified > fileAge); 363 } 364 365 protected boolean hasOsmTileETag(String eTag) throws IOException { 366 URL url; 367 url = new URL(tile.getUrl()); 368 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 369 prepareHttpUrlConnection(urlConn); 370 urlConn.setRequestMethod("HEAD"); 371 urlConn.setReadTimeout(30000); // 30 seconds read timeout 372 // System.out.println("Tile age: " + new 373 // Date(urlConn.getLastModified()) + " / " 374 // + new Date(fileAge)); 375 String osmETag = urlConn.getHeaderField("ETag"); 376 if (osmETag == null) 377 return true; 378 return (osmETag.equals(eTag)); 379 } 380 381 protected File getTileFile() { 382 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." 383 + tile.getSource().getTileType()); 384 } 385 386 protected File getTagsFile() { 387 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() 388 + TAGS_FILE_EXT); 389 } 390 391 protected void saveTileToFile(byte[] rawData) { 392 try { 393 FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() 394 + "_" + tile.getYtile() + "." + tile.getSource().getTileType()); 395 f.write(rawData); 396 f.close(); 397 // System.out.println("Saved tile to file: " + tile); 398 } catch (Exception e) { 399 System.err.println("Failed to save tile content: " + e.getLocalizedMessage()); 400 } 401 } 402 403 protected void saveTagsToFile() { 404 File tagsFile = getTagsFile(); 405 if (tile.getMetadata() == null) { 406 tagsFile.delete(); 407 return; 408 } 409 try { 410 final PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), 411 TAGS_CHARSET)); 412 for (Entry<String, String> entry : tile.getMetadata().entrySet()) { 413 f.println(entry.getKey() + "=" + entry.getValue()); 414 } 415 f.close(); 416 } catch (Exception e) { 417 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); 418 } 419 } 420 421 /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/ 422 private void loadOldETagfromFile() { 423 File etagFile = new File(tileCacheDir, tile.getZoom() + "_" 424 + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT); 425 if (!etagFile.exists()) return; 426 try { 427 FileInputStream f = new FileInputStream(etagFile); 428 byte[] buf = new byte[f.available()]; 429 f.read(buf); 430 f.close(); 431 String etag = new String(buf, TAGS_CHARSET.name()); 432 tile.putValue("etag", etag); 433 if (etagFile.delete()) { 434 saveTagsToFile(); 435 } 436 } catch (IOException e) { 437 System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage()); 438 } 439 } 440 441 protected void loadTagsFromFile() { 442 loadOldETagfromFile(); 443 File tagsFile = getTagsFile(); 444 try { 445 final BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), 446 TAGS_CHARSET)); 447 for (String line = f.readLine(); line != null; line = f.readLine()) { 448 final int i = line.indexOf('='); 449 if (i == -1 || i == 0) { 450 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); 451 continue; 452 } 453 tile.putValue(line.substring(0,i),line.substring(i+1)); 454 } 455 f.close(); 456 } catch (FileNotFoundException e) { 457 } catch (Exception e) { 458 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); 459 } 460 } 461 462 } 463 464 public long getMaxFileAge() { 465 return maxCacheFileAge; 466 } 467 468 /** 469 * Sets the maximum age of the local cached tile in the file system. If a 470 * local tile is older than the specified file age 471 * {@link OsmFileCacheTileLoader} will connect to the tile server and check 472 * if a newer tile is available using the mechanism specified for the 473 * selected tile source/server. 474 * 475 * @param maxFileAge 476 * maximum age in milliseconds 477 * @see #FILE_AGE_ONE_DAY 478 * @see #FILE_AGE_ONE_WEEK 479 * @see TileSource#getTileUpdate() 480 */ 481 public void setCacheMaxFileAge(long maxFileAge) { 482 this.maxCacheFileAge = maxFileAge; 483 } 484 485 public String getCacheDirBase() { 486 return cacheDirBase; 487 } 488 489 public void setTileCacheDir(String tileCacheDir) { 490 File dir = new File(tileCacheDir); 491 dir.mkdirs(); 492 this.cacheDirBase = dir.getAbsolutePath(); 493 } 494 495 @Override 496 public void clearCache(TileSource source) { 497 clearCache(source, null); 498 } 499 500 @Override 501 public void clearCache(TileSource source, TileClearController controller) { 502 File dir = getSourceCacheDir(source); 503 if (dir != null) { 504 if (controller != null) controller.initClearDir(dir); 505 if (dir.isDirectory()) { 506 File[] files = dir.listFiles(); 507 if (controller != null) controller.initClearFiles(files); 508 for (File file : files) { 509 if (controller != null && controller.cancel()) return; 510 file.delete(); 511 if (controller != null) controller.fileDeleted(file); 512 } 513 } 514 dir.delete(); 515 } 516 if (controller != null) controller.clearFinished(); 517 } 518}