001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.net.HttpURLConnection; 014import java.net.MalformedURLException; 015import java.net.URL; 016import java.util.ArrayList; 017import java.util.Arrays; 018import java.util.Enumeration; 019import java.util.List; 020import java.util.zip.ZipEntry; 021import java.util.zip.ZipFile; 022 023import org.openstreetmap.josm.Main; 024import org.openstreetmap.josm.tools.Pair; 025import org.openstreetmap.josm.tools.Utils; 026 027/** 028 * Mirrors a file to a local file. 029 * <p> 030 * The file mirrored is only downloaded if it has been more than 7 days since last download 031 */ 032public class MirroredInputStream extends InputStream { 033 InputStream fs = null; 034 File file = null; 035 036 public final static long DEFAULT_MAXTIME = -1L; 037 038 public MirroredInputStream(String name) throws IOException { 039 this(name, null, DEFAULT_MAXTIME); 040 } 041 042 public MirroredInputStream(String name, long maxTime) throws IOException { 043 this(name, null, maxTime); 044 } 045 046 public MirroredInputStream(String name, String destDir) throws IOException { 047 this(name, destDir, DEFAULT_MAXTIME); 048 } 049 050 /** 051 * Get an inputstream from a given filename, url or internal resource. 052 * @param name can be 053 * - relative or absolute file name 054 * - file:///SOME/FILE the same as above 055 * - resource://SOME/FILE file from the classpath (usually in the current *.jar) 056 * - http://... a url. It will be cached on disk. 057 * @param destDir the destination directory for the cache file. only applies for urls. 058 * @param maxTime the maximum age of the cache file (in seconds) 059 * @throws IOException when the resource with the given name could not be retrieved 060 */ 061 public MirroredInputStream(String name, String destDir, long maxTime) throws IOException { 062 URL url; 063 try { 064 url = new URL(name); 065 if (url.getProtocol().equals("file")) { 066 file = new File(name.substring("file:/".length())); 067 if (!file.exists()) { 068 file = new File(name.substring("file://".length())); 069 } 070 } else { 071 if (Main.applet) { 072 fs = new BufferedInputStream(Utils.openURL(url)); 073 file = new File(url.getFile()); 074 } else { 075 file = checkLocal(url, destDir, maxTime); 076 } 077 } 078 } catch (java.net.MalformedURLException e) { 079 if (name.startsWith("resource://")) { 080 fs = getClass().getResourceAsStream( 081 name.substring("resource:/".length())); 082 if (fs == null) 083 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 084 return; 085 } 086 file = new File(name); 087 } 088 if (file == null) 089 throw new IOException(); 090 fs = new FileInputStream(file); 091 } 092 093 /** 094 * Looks for a certain entry inside a zip file and returns the entry path. 095 * 096 * Replies a file in the top level directory of the ZIP file which has an 097 * extension <code>extension</code>. If more than one files have this 098 * extension, the last file whose name includes <code>namepart</code> 099 * is opened. 100 * 101 * @param extension the extension of the file we're looking for 102 * @param namepart the name part 103 * @return The zip entry path of the matching file. Null if this mirrored 104 * input stream doesn't represent a zip file or if there was no matching 105 * file in the ZIP file. 106 */ 107 public String findZipEntryPath(String extension, String namepart) { 108 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 109 if (ze == null) return null; 110 return ze.a; 111 } 112 113 /** 114 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 115 */ 116 public InputStream findZipEntryInputStream(String extension, String namepart) { 117 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 118 if (ze == null) return null; 119 return ze.b; 120 } 121 122 @Deprecated // use findZipEntryInputStream 123 public InputStream getZipEntry(String extension, String namepart) { 124 return findZipEntryInputStream(extension, namepart); 125 } 126 127 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 128 if (file == null) 129 return null; 130 Pair<String, InputStream> res = null; 131 try { 132 ZipFile zipFile = new ZipFile(file); 133 ZipEntry resentry = null; 134 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 135 while (entries.hasMoreElements()) { 136 ZipEntry entry = entries.nextElement(); 137 if (entry.getName().endsWith("." + extension)) { 138 /* choose any file with correct extension. When more than 139 one file, prefer the one which matches namepart */ 140 if (resentry == null || entry.getName().indexOf(namepart) >= 0) { 141 resentry = entry; 142 } 143 } 144 } 145 if (resentry != null) { 146 InputStream is = zipFile.getInputStream(resentry); 147 res = Pair.create(resentry.getName(), is); 148 } else { 149 Utils.close(zipFile); 150 } 151 } catch (Exception e) { 152 if (file.getName().endsWith(".zip")) { 153 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 154 file.getName(), e.toString(), extension, namepart)); 155 } 156 } 157 return res; 158 } 159 160 public File getFile() { 161 return file; 162 } 163 164 public static void cleanup(String name) { 165 cleanup(name, null); 166 } 167 168 public static void cleanup(String name, String destDir) { 169 URL url; 170 try { 171 url = new URL(name); 172 if (!url.getProtocol().equals("file")) { 173 String prefKey = getPrefKey(url, destDir); 174 List<String> localPath = new ArrayList<String>(Main.pref.getCollection(prefKey)); 175 if (localPath.size() == 2) { 176 File lfile = new File(localPath.get(1)); 177 if(lfile.exists()) { 178 lfile.delete(); 179 } 180 } 181 Main.pref.putCollection(prefKey, null); 182 } 183 } catch (MalformedURLException e) { 184 Main.warn(e); 185 } 186 } 187 188 /** 189 * get preference key to store the location and age of the cached file. 190 * 2 resources that point to the same url, but that are to be stored in different 191 * directories will not share a cache file. 192 */ 193 private static String getPrefKey(URL url, String destDir) { 194 StringBuilder prefKey = new StringBuilder("mirror."); 195 if (destDir != null) { 196 prefKey.append(destDir); 197 prefKey.append("."); 198 } 199 prefKey.append(url.toString()); 200 return prefKey.toString().replaceAll("=","_"); 201 } 202 203 private File checkLocal(URL url, String destDir, long maxTime) throws IOException { 204 String prefKey = getPrefKey(url, destDir); 205 long age = 0L; 206 File localFile = null; 207 List<String> localPathEntry = new ArrayList<String>(Main.pref.getCollection(prefKey)); 208 if (localPathEntry.size() == 2) { 209 localFile = new File(localPathEntry.get(1)); 210 if(!localFile.exists()) 211 localFile = null; 212 else { 213 if ( maxTime == DEFAULT_MAXTIME 214 || maxTime <= 0 // arbitrary value <= 0 is deprecated 215 ) { 216 maxTime = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week 217 } 218 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 219 if (age < maxTime*1000) { 220 return localFile; 221 } 222 } 223 } 224 if (destDir == null) { 225 destDir = Main.pref.getCacheDirectory().getPath(); 226 } 227 228 File destDirFile = new File(destDir); 229 if (!destDirFile.exists()) { 230 destDirFile.mkdirs(); 231 } 232 233 String a = url.toString().replaceAll("[^A-Za-z0-9_.-]", "_"); 234 String localPath = "mirror_" + a; 235 destDirFile = new File(destDir, localPath + ".tmp"); 236 BufferedOutputStream bos = null; 237 BufferedInputStream bis = null; 238 try { 239 HttpURLConnection con = connectFollowingRedirect(url); 240 bis = new BufferedInputStream(con.getInputStream()); 241 FileOutputStream fos = new FileOutputStream(destDirFile); 242 bos = new BufferedOutputStream(fos); 243 byte[] buffer = new byte[4096]; 244 int length; 245 while ((length = bis.read(buffer)) > -1) { 246 bos.write(buffer, 0, length); 247 } 248 Utils.close(bos); 249 bos = null; 250 /* close fos as well to be sure! */ 251 Utils.close(fos); 252 fos = null; 253 localFile = new File(destDir, localPath); 254 if(Main.platform.rename(destDirFile, localFile)) { 255 Main.pref.putCollection(prefKey, Arrays.asList(new String[] 256 {Long.toString(System.currentTimeMillis()), localFile.toString()})); 257 } else { 258 Main.warn(tr("Failed to rename file {0} to {1}.", 259 destDirFile.getPath(), localFile.getPath())); 260 } 261 } catch (IOException e) { 262 if (age >= maxTime*1000 && age < maxTime*1000*2) { 263 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", url, e)); 264 return localFile; 265 } else { 266 throw e; 267 } 268 } finally { 269 Utils.close(bis); 270 Utils.close(bos); 271 } 272 273 return localFile; 274 } 275 276 /** 277 * Opens a connection for downloading a resource. 278 * <p> 279 * Manually follows redirects because 280 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect 281 * is going from a http to a https URL, see <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4620571">bug report</a>. 282 * <p> 283 * This can causes problems when downloading from certain GitHub URLs. 284 * 285 * @param downloadUrl The resource URL to download 286 * @return The HTTP connection effectively linked to the resource, after all potential redirections 287 * @throws MalformedURLException If a redirected URL is wrong 288 * @throws IOException If any I/O operation goes wrong 289 * @since 6073 290 */ 291 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl) throws MalformedURLException, IOException { 292 HttpURLConnection con = null; 293 int numRedirects = 0; 294 while(true) { 295 con = Utils.openHttpConnection(downloadUrl); 296 con.setInstanceFollowRedirects(false); 297 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000); 298 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000); 299 con.connect(); 300 switch(con.getResponseCode()) { 301 case HttpURLConnection.HTTP_OK: 302 return con; 303 case HttpURLConnection.HTTP_MOVED_PERM: 304 case HttpURLConnection.HTTP_MOVED_TEMP: 305 case HttpURLConnection.HTTP_SEE_OTHER: 306 String redirectLocation = con.getHeaderField("Location"); 307 if (downloadUrl == null) { 308 /* I18n: argument is HTTP response code */ String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header. Can''t redirect. Aborting.", con.getResponseCode()); 309 throw new IOException(msg); 310 } 311 downloadUrl = new URL(redirectLocation); 312 // keep track of redirect attempts to break a redirect loops if it happens 313 // to occur for whatever reason 314 numRedirects++; 315 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) { 316 String msg = tr("Too many redirects to the download URL detected. Aborting."); 317 throw new IOException(msg); 318 } 319 Main.info(tr("Download redirected to ''{0}''", downloadUrl)); 320 break; 321 default: 322 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadUrl, con.getResponseCode()); 323 throw new IOException(msg); 324 } 325 } 326 } 327 328 @Override 329 public int available() throws IOException 330 { return fs.available(); } 331 @Override 332 public void close() throws IOException 333 { Utils.close(fs); } 334 @Override 335 public int read() throws IOException 336 { return fs.read(); } 337 @Override 338 public int read(byte[] b) throws IOException 339 { return fs.read(b); } 340 @Override 341 public int read(byte[] b, int off, int len) throws IOException 342 { return fs.read(b,off, len); } 343 @Override 344 public long skip(long n) throws IOException 345 { return fs.skip(n); } 346}