001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.File; 008import java.io.FileInputStream; 009import java.io.IOException; 010import java.io.InputStream; 011import java.net.HttpURLConnection; 012import java.net.MalformedURLException; 013import java.net.URL; 014import java.nio.charset.StandardCharsets; 015import java.nio.file.Files; 016import java.nio.file.StandardCopyOption; 017import java.util.ArrayList; 018import java.util.Arrays; 019import java.util.Enumeration; 020import java.util.List; 021import java.util.Map; 022import java.util.concurrent.ConcurrentHashMap; 023import java.util.zip.ZipEntry; 024import java.util.zip.ZipFile; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.tools.HttpClient; 028import org.openstreetmap.josm.tools.Pair; 029import org.openstreetmap.josm.tools.Utils; 030 031/** 032 * Downloads a file and caches it on disk in order to reduce network load. 033 * 034 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 035 * resources from the current *.jar file. (Local caching is only done for URLs.) 036 * <p> 037 * The mirrored file is only downloaded if it has been more than 7 days since 038 * last download. (Time can be configured.) 039 * <p> 040 * The file content is normally accessed with {@link #getInputStream()}, but 041 * you can also get the mirrored copy with {@link #getFile()}. 042 */ 043public class CachedFile { 044 045 /** 046 * Caching strategy. 047 */ 048 public enum CachingStrategy { 049 /** 050 * If cached file on disk is older than a certain time (7 days by default), 051 * consider the cache stale and try to download the file again. 052 */ 053 MaxAge, 054 /** 055 * Similar to MaxAge, considers the cache stale when a certain age is 056 * exceeded. In addition, a If-Modified-Since HTTP header is added. 057 * When the server replies "304 Not Modified", this is considered the same 058 * as a full download. 059 */ 060 IfModifiedSince 061 } 062 063 protected String name; 064 protected long maxAge; 065 protected String destDir; 066 protected String httpAccept; 067 protected CachingStrategy cachingStrategy; 068 069 protected File cacheFile; 070 protected boolean initialized; 071 072 public static final long DEFAULT_MAXTIME = -1L; 073 public static final long DAYS = 24*60*60; // factor to get caching time in days 074 075 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 076 077 /** 078 * Constructs a CachedFile object from a given filename, URL or internal resource. 079 * 080 * @param name can be:<ul> 081 * <li>relative or absolute file name</li> 082 * <li>{@code file:///SOME/FILE} the same as above</li> 083 * <li>{@code http://...} a URL. It will be cached on disk.</li> 084 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 085 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 086 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 087 */ 088 public CachedFile(String name) { 089 this.name = name; 090 } 091 092 /** 093 * Set the name of the resource. 094 * @param name can be:<ul> 095 * <li>relative or absolute file name</li> 096 * <li>{@code file:///SOME/FILE} the same as above</li> 097 * <li>{@code http://...} a URL. It will be cached on disk.</li> 098 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 099 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 100 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 101 * @return this object 102 */ 103 public CachedFile setName(String name) { 104 this.name = name; 105 return this; 106 } 107 108 /** 109 * Set maximum age of cache file. Only applies to URLs. 110 * When this time has passed after the last download of the file, the 111 * cache is considered stale and a new download will be attempted. 112 * @param maxAge the maximum cache age in seconds 113 * @return this object 114 */ 115 public CachedFile setMaxAge(long maxAge) { 116 this.maxAge = maxAge; 117 return this; 118 } 119 120 /** 121 * Set the destination directory for the cache file. Only applies to URLs. 122 * @param destDir the destination directory 123 * @return this object 124 */ 125 public CachedFile setDestDir(String destDir) { 126 this.destDir = destDir; 127 return this; 128 } 129 130 /** 131 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 132 * @param httpAccept the accepted MIME types 133 * @return this object 134 */ 135 public CachedFile setHttpAccept(String httpAccept) { 136 this.httpAccept = httpAccept; 137 return this; 138 } 139 140 /** 141 * Set the caching strategy. Only applies to URLs. 142 * @param cachingStrategy caching strategy 143 * @return this object 144 */ 145 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 146 this.cachingStrategy = cachingStrategy; 147 return this; 148 } 149 150 /** 151 * Sets the http headers. Only applies to URL pointing to http or https resources 152 * @param headers that should be sent together with request 153 * @return this object 154 */ 155 public CachedFile setHttpHeaders(Map<String, String> headers) { 156 this.httpHeaders.putAll(headers); 157 return this; 158 } 159 160 public String getName() { 161 return name; 162 } 163 164 public long getMaxAge() { 165 return maxAge; 166 } 167 168 public String getDestDir() { 169 return destDir; 170 } 171 172 public String getHttpAccept() { 173 return httpAccept; 174 } 175 176 public CachingStrategy getCachingStrategy() { 177 return cachingStrategy; 178 } 179 180 /** 181 * Get InputStream to the requested resource. 182 * @return the InputStream 183 * @throws IOException when the resource with the given name could not be retrieved 184 */ 185 public InputStream getInputStream() throws IOException { 186 File file = getFile(); 187 if (file == null) { 188 if (name.startsWith("resource://")) { 189 InputStream is = getClass().getResourceAsStream( 190 name.substring("resource:/".length())); 191 if (is == null) 192 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 193 return is; 194 } else { 195 throw new IOException("No file found for: "+name); 196 } 197 } 198 return new FileInputStream(file); 199 } 200 201 /** 202 * Get local file for the requested resource. 203 * @return The local cache file for URLs. If the resource is a local file, 204 * returns just that file. 205 * @throws IOException when the resource with the given name could not be retrieved 206 */ 207 public synchronized File getFile() throws IOException { 208 if (initialized) 209 return cacheFile; 210 initialized = true; 211 URL url; 212 try { 213 url = new URL(name); 214 if ("file".equals(url.getProtocol())) { 215 cacheFile = new File(name.substring("file:/".length() - 1)); 216 if (!cacheFile.exists()) { 217 cacheFile = new File(name.substring("file://".length() - 1)); 218 } 219 } else { 220 cacheFile = checkLocal(url); 221 } 222 } catch (MalformedURLException e) { 223 if (name.startsWith("resource://")) { 224 return null; 225 } else if (name.startsWith("josmdir://")) { 226 cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length())); 227 } else if (name.startsWith("josmplugindir://")) { 228 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 229 } else { 230 cacheFile = new File(name); 231 } 232 } 233 if (cacheFile == null) 234 throw new IOException("Unable to get cache file for "+name); 235 return cacheFile; 236 } 237 238 /** 239 * Looks for a certain entry inside a zip file and returns the entry path. 240 * 241 * Replies a file in the top level directory of the ZIP file which has an 242 * extension <code>extension</code>. If more than one files have this 243 * extension, the last file whose name includes <code>namepart</code> 244 * is opened. 245 * 246 * @param extension the extension of the file we're looking for 247 * @param namepart the name part 248 * @return The zip entry path of the matching file. Null if this cached file 249 * doesn't represent a zip file or if there was no matching 250 * file in the ZIP file. 251 */ 252 public String findZipEntryPath(String extension, String namepart) { 253 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 254 if (ze == null) return null; 255 return ze.a; 256 } 257 258 /** 259 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 260 * @param extension the extension of the file we're looking for 261 * @param namepart the name part 262 * @return InputStream to the matching file. Null if this cached file 263 * doesn't represent a zip file or if there was no matching 264 * file in the ZIP file. 265 * @since 6148 266 */ 267 public InputStream findZipEntryInputStream(String extension, String namepart) { 268 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 269 if (ze == null) return null; 270 return ze.b; 271 } 272 273 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 274 File file = null; 275 try { 276 file = getFile(); 277 } catch (IOException ex) { 278 Main.warn(ex, false); 279 } 280 if (file == null) 281 return null; 282 Pair<String, InputStream> res = null; 283 try { 284 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 285 ZipEntry resentry = null; 286 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 287 while (entries.hasMoreElements()) { 288 ZipEntry entry = entries.nextElement(); 289 if (entry.getName().endsWith('.' + extension)) { 290 /* choose any file with correct extension. When more than 291 one file, prefer the one which matches namepart */ 292 if (resentry == null || entry.getName().indexOf(namepart) >= 0) { 293 resentry = entry; 294 } 295 } 296 } 297 if (resentry != null) { 298 InputStream is = zipFile.getInputStream(resentry); 299 res = Pair.create(resentry.getName(), is); 300 } else { 301 Utils.close(zipFile); 302 } 303 } catch (Exception e) { 304 if (file.getName().endsWith(".zip")) { 305 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 306 file.getName(), e.toString(), extension, namepart)); 307 } 308 } 309 return res; 310 } 311 312 /** 313 * Clear the cache for the given resource. 314 * This forces a fresh download. 315 * @param name the URL 316 */ 317 public static void cleanup(String name) { 318 cleanup(name, null); 319 } 320 321 /** 322 * Clear the cache for the given resource. 323 * This forces a fresh download. 324 * @param name the URL 325 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 326 */ 327 public static void cleanup(String name, String destDir) { 328 URL url; 329 try { 330 url = new URL(name); 331 if (!"file".equals(url.getProtocol())) { 332 String prefKey = getPrefKey(url, destDir); 333 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey)); 334 if (localPath.size() == 2) { 335 File lfile = new File(localPath.get(1)); 336 if (lfile.exists()) { 337 Utils.deleteFile(lfile); 338 } 339 } 340 Main.pref.putCollection(prefKey, null); 341 } 342 } catch (MalformedURLException e) { 343 Main.warn(e); 344 } 345 } 346 347 /** 348 * Get preference key to store the location and age of the cached file. 349 * 2 resources that point to the same url, but that are to be stored in different 350 * directories will not share a cache file. 351 * @param url URL 352 * @param destDir destination directory 353 * @return Preference key 354 */ 355 private static String getPrefKey(URL url, String destDir) { 356 StringBuilder prefKey = new StringBuilder("mirror."); 357 if (destDir != null) { 358 prefKey.append(destDir).append('.'); 359 } 360 prefKey.append(url.toString()); 361 return prefKey.toString().replaceAll("=", "_"); 362 } 363 364 private File checkLocal(URL url) throws IOException { 365 String prefKey = getPrefKey(url, destDir); 366 String urlStr = url.toExternalForm(); 367 long age = 0L; 368 long lMaxAge = maxAge; 369 Long ifModifiedSince = null; 370 File localFile = null; 371 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey)); 372 boolean offline = false; 373 try { 374 checkOfflineAccess(urlStr); 375 } catch (OfflineAccessException e) { 376 offline = true; 377 } 378 if (localPathEntry.size() == 2) { 379 localFile = new File(localPathEntry.get(1)); 380 if (!localFile.exists()) { 381 localFile = null; 382 } else { 383 if (maxAge == DEFAULT_MAXTIME 384 || maxAge <= 0 // arbitrary value <= 0 is deprecated 385 ) { 386 lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week 387 } 388 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 389 if (offline || age < lMaxAge*1000) { 390 return localFile; 391 } 392 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 393 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 394 } 395 } 396 } 397 if (destDir == null) { 398 destDir = Main.pref.getCacheDirectory().getPath(); 399 } 400 401 File destDirFile = new File(destDir); 402 if (!destDirFile.exists()) { 403 destDirFile.mkdirs(); 404 } 405 406 // No local file + offline => nothing to do 407 if (offline) { 408 return null; 409 } 410 411 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 412 String localPath = "mirror_" + a; 413 destDirFile = new File(destDir, localPath + ".tmp"); 414 try { 415 final HttpClient.Response con = HttpClient.create(url) 416 .setAccept(httpAccept) 417 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 418 .setHeaders(httpHeaders) 419 .connect(); 420 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 421 if (Main.isDebugEnabled()) { 422 Main.debug("304 Not Modified ("+urlStr+')'); 423 } 424 if (localFile == null) 425 throw new AssertionError(); 426 Main.pref.putCollection(prefKey, 427 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 428 return localFile; 429 } 430 try (InputStream bis = new BufferedInputStream(con.getContent())) { 431 Files.copy(bis, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 432 } 433 localFile = new File(destDir, localPath); 434 if (Main.platform.rename(destDirFile, localFile)) { 435 Main.pref.putCollection(prefKey, 436 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 437 } else { 438 Main.warn(tr("Failed to rename file {0} to {1}.", 439 destDirFile.getPath(), localFile.getPath())); 440 } 441 } catch (IOException e) { 442 if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) { 443 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 444 return localFile; 445 } else { 446 throw e; 447 } 448 } 449 450 return localFile; 451 } 452 453 private static void checkOfflineAccess(String urlString) { 454 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 455 OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL)); 456 } 457 458}