1 package de.spiritscorp.datasync.model;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 import java.io.IOException;
24 import java.nio.file.Files;
25 import java.nio.file.Path;
26 import java.nio.file.StandardCopyOption;
27 import java.util.ArrayList;
28 import java.util.Collections;
29 import java.util.HashMap;
30 import java.util.HashSet;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.ExecutorService;
34 import java.util.concurrent.Executors;
35 import java.util.concurrent.TimeUnit;
36
37 import de.spiritscorp.datasync.ScanType;
38 import de.spiritscorp.datasync.io.Debug;
39 import de.spiritscorp.datasync.io.Logger;
40
41 class FileHandler {
42
43 private final Logger log;
44 private final int avgProc;
45
46 FileHandler( Logger log ) {
47 this.log = log;
48 avgProc = ( Runtime.getRuntime().availableProcessors() > 3 ) ? ( Runtime.getRuntime().availableProcessors() / 2 ) - 1 : 1;
49 }
50
51
52
53
54
55
56
57
58
59
60 void listFiles( ArrayList<Path> paths, Map<Path, FileAttributes> resultMap, ScanType deepScan, boolean subDir ) {
61 final ExecutorService executor = Executors.newSingleThreadExecutor();
62 for( final Path path : paths ) {
63
64 if( Thread.currentThread().isInterrupted() ) {
65 Debug.printDebug( "[FileHandler] Interruption detected prior to walking directory path: %s", path.toString() );
66 executor.shutdownNow();
67 return;
68 }
69 if( Files.exists( path ) ) {
70 try {
71 final Path baseDir = subDir ? path.getParent() : path;
72 Files.walkFileTree( path, new FileVisit( executor, baseDir, resultMap, deepScan ) );
73 }catch( final IOException e ) {
74 Debug.printException( this.getClass(), e );
75 }
76 }
77 }
78 executor.shutdown();
79 try {
80 while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
81 if( Thread.currentThread().isInterrupted() ) {
82 executor.shutdownNow();
83 throw new InterruptedException();
84 }
85 }
86 }catch( final InterruptedException e ) {
87 Debug.printDebug( "[FileHandler] File processing walk subsystem was forcefully interrupted." );
88 Thread.currentThread().interrupt();
89 }
90 for( final Path path : paths ) {
91 Debug.printDebug( "[FileHandler] ListFiles() -> ready %s -> %s", Thread.currentThread().getName(), path.toString() );
92 }
93 }
94
95
96
97
98
99
100
101
102 Map<Path, FileAttributes> findDuplicates( Map<Path, FileAttributes> sourceMap ) {
103 Debug.printDebug( "[FileHandler] entryPaths -> %d", sourceMap.size() );
104 final Map<Path, FileAttributes> duplicateMap = Model.createMap();
105
106 final Map<Long, ArrayList<Path>> mapSize = new HashMap<>();
107 for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
108 if( Thread.currentThread().isInterrupted() ) return duplicateMap;
109 final long size = entry.getValue().getSize();
110 if( mapSize.containsKey( size ) ) {
111 mapSize.get( size ).add( entry.getKey() );
112 }else {
113 mapSize.put( size, new ArrayList<>() );
114 mapSize.get( size ).add( entry.getKey() );
115 }
116 }
117
118 for( final Map.Entry<Long, ArrayList<Path>> entry : mapSize.entrySet() ) {
119 if( Thread.currentThread().isInterrupted() ) return duplicateMap;
120 final ArrayList<Path> paths = entry.getValue();
121 if( paths.size() > 1 ) {
122 for( int i = 0; i < paths.size(); i++ ) {
123 final String firstPath = sourceMap.get( paths.get( i ) ).getFileHash();
124 for( int j = i + 1; j < paths.size(); j++ ) {
125 if( firstPath.equals( sourceMap.get( paths.get( j ) ).getFileHash() ) ) {
126 duplicateMap.put( paths.get( i ), sourceMap.get( paths.get( i ) ) );
127 duplicateMap.put( paths.get( j ), sourceMap.get( paths.get( j ) ) );
128 }
129 }
130 }
131 }
132 }
133 Debug.printDebug( "[FileHandler] DuplicateList -> ready : size: %d", duplicateMap.size() );
134 return duplicateMap;
135 }
136
137
138
139
140
141
142
143 void equalsFiles( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap ) {
144 Debug.printDebug( "[FileHandler] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
145 if( sourceMap.size() != 0 && destMap.size() != 0 ) {
146 final Set<Path> sourceHitList = Collections.synchronizedSet( new HashSet<>() );
147 final Set<Path> destHitList = Collections.synchronizedSet( new HashSet<>() );
148 if( sourceMap.size() > 30_000 ) {
149 final ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 );
150 final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
151 final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
152 for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
153 executor.execute( () -> equalsMap( source.getValue(), destMap, sourceHitList ) );
154 }
155 for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
156 executor.execute( () -> equalsMap( dest.getValue(), sourceMap, destHitList ) );
157 }
158 executor.shutdown();
159 try {
160 while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
161 if( Thread.currentThread().isInterrupted() ) {
162 executor.shutdownNow();
163 return;
164 }
165 }
166 }catch( final InterruptedException e ) {
167 executor.shutdownNow();
168 Thread.currentThread().interrupt();
169 return;
170 }
171 }else {
172 final Thread t1 = new Thread( () -> equalsMap( sourceMap, destMap, sourceHitList ) );
173 final Thread t2 = new Thread( () -> equalsMap( destMap, sourceMap, destHitList ) );
174 t1.start();
175 t2.start();
176 try {
177 t1.join();
178 t2.join();
179 }catch( final InterruptedException e ) {
180 t1.interrupt();
181 t2.interrupt();
182 Thread.currentThread().interrupt();
183 return;
184 }
185 }
186
187 if( Thread.currentThread().isInterrupted() ) return;
188 for( final Path p : sourceHitList ) {
189 sourceMap.remove( p );
190 }
191 for( final Path p : destHitList ) {
192 destMap.remove( p );
193 }
194 Debug.printDebug( "[FileHandler] Full source hitList size: %d && Full destination hitList size: %d", sourceMap.size(), destMap.size() );
195 }
196 }
197
198
199
200
201
202
203
204
205
206
207
208
209
210 ArrayList<Map<Path, FileAttributes>> getSyncFiles( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap, Path startSourcePath, Path startDestPath,
211 Map<Path, FileAttributes> syncMap ) {
212 Debug.printDebug( "[FileHandler] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
213 final ArrayList<Map<Path, FileAttributes>> resultValue = new ArrayList<>();
214 final ArrayList<Map<Path, FileAttributes>> destValue = new ArrayList<>();
215 final Map<Path, FileAttributes> copySourceHitList = Model.createMap();
216 final Map<Path, FileAttributes> copyDestHitList = Model.createMap();
217 final Map<Path, FileAttributes> delHitList = Model.createMap();
218
219 resultValue.add( copySourceHitList );
220 resultValue.add( copyDestHitList );
221 resultValue.add( delHitList );
222 destValue.add( copyDestHitList );
223 destValue.add( copySourceHitList );
224 destValue.add( delHitList );
225 if( sourceMap.size() > 0 || destMap.size() > 0 ) {
226 if( sourceMap.size() > 30_000 || destMap.size() > 30_000 ) {
227 final ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 );
228 final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
229 final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
230
231 for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
232 executor.execute( () -> syncMaps( source.getValue(), destMap, resultValue, startDestPath, syncMap ) );
233 }
234 for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
235 executor.execute( () -> syncMaps( dest.getValue(), sourceMap, destValue, startSourcePath, syncMap ) );
236 }
237 executor.shutdown();
238 try {
239 while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
240 if( Thread.currentThread().isInterrupted() ) {
241 executor.shutdownNow();
242 return resultValue;
243 }
244 }
245 }catch( final InterruptedException e ) {
246 executor.shutdownNow();
247 Thread.currentThread().interrupt();
248 return resultValue;
249 }
250 }else {
251 syncMaps( sourceMap, destMap, resultValue, startDestPath, syncMap );
252 if( Thread.currentThread().isInterrupted() ) return resultValue;
253 syncMaps( destMap, sourceMap, destValue, startSourcePath, syncMap );
254 }
255 }
256 Debug.printDebug( "[FileHandler] Full copySourceHitList size: %d && Full copyDestHitList size: %d && Full delHitList size: %d",
257 copySourceHitList.size(), copyDestHitList.size(), delHitList.size() );
258 return resultValue;
259 }
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278 void deleteFiles( Map<Path, FileAttributes> map, boolean logOn, boolean trashbin, Path trashbinPath ) {
279 for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
280
281 if( Thread.currentThread().isInterrupted() ) {
282 Debug.printDebug( "[FileHandler] Safe loop interruption caught within file deletion loop vector." );
283 break;
284 }
285
286 final FileAttributes fileAttr = entry.getValue();
287 final Path path = entry.getKey();
288 if( fileAttr == null ) continue;
289 try {
290 if( trashbin && trashbinPath != null ) {
291 Files.createDirectories( trashbinPath.resolve( fileAttr.getRelativeFilePath() ) );
292 Files.copy( path, trashbinPath.resolve( fileAttr.getRelativeFilePath() ), StandardCopyOption.REPLACE_EXISTING );
293 }
294 boolean writable = true;
295 if( !path.toFile().canWrite() ) writable = path.toFile().setWritable( true );
296 if( writable ) {
297 Files.delete( path );
298 log.setEntry( path.toString(), "gelöscht", fileAttr );
299 }else {
300 log.setEntry( path.toString(), "SCHREIBSCHUTZ BEIM LÖSCHEN", fileAttr );
301 Debug.printDebug( "[FileHandler Error] Delete failed, target file is not writable: %s", path.toString() );
302 }
303 }catch( final IOException e ) {
304 log.setEntry( path.toString(), "FEHLER BEIM LÖSCHEN", fileAttr );
305 Debug.printDebug( "[FileHandler Error] Delete failed: %s", path.toString() );
306 Debug.printException( this.getClass(), e );
307 }
308 }
309 map.clear();
310 if( logOn ) log.printStatus();
311 }
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330 void copyFiles( Map<Path, FileAttributes> map, boolean logOn, Path destPath ) {
331 for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
332
333 if( Thread.currentThread().isInterrupted() ) {
334 Debug.printDebug( "[FileHandler] Safe loop interruption caught within file replication loop vector." );
335 break;
336 }
337 final FileAttributes fileAttr = entry.getValue();
338 if( fileAttr == null ) continue;
339 final Path path = destPath.resolve( fileAttr.getRelativeFilePath() );
340 final Path parentPath = path.getParent();
341 boolean writable = true;
342 try {
343 if( parentPath != null && !Files.exists( parentPath ) )
344 Files.createDirectories( parentPath );
345 else if( Files.exists( path ) && !path.toFile().canWrite() ) writable = path.toFile().setWritable( true );
346 if( writable ) {
347 Files.copy(
348 entry.getKey(),
349 path,
350 StandardCopyOption.REPLACE_EXISTING,
351 StandardCopyOption.COPY_ATTRIBUTES );
352 Files.setAttribute( path, "creationTime", fileAttr.getCreateTime() );
353 log.setEntry( path.toString(), "kopiert", fileAttr );
354 }else {
355 log.setEntry( path.toString(), "SCHREIBSCHUTZ BEIM KOPIEREN", fileAttr );
356 Debug.printDebug( "[FileHandler Error] Copy failed, target file is not writable: %s", path.toString() );
357 }
358 }catch( final IOException e ) {
359 log.setEntry( path.toString(), "FEHLER BEIM KOPIEREN", fileAttr );
360 Debug.printDebug( "[FileHandler Error] Copy failed: %s", path.toString() );
361 Debug.printException( this.getClass(), e );
362 }
363 }
364 map.clear();
365 if( logOn ) log.printStatus();
366 }
367
368
369
370
371
372
373
374
375
376
377 private void equalsMap( Map<Path, FileAttributes> iterateMap, Map<Path, FileAttributes> fullMap, Set<Path> hitList ) {
378 for( final Map.Entry<Path, FileAttributes> entry : iterateMap.entrySet() ) {
379 if( Thread.currentThread().isInterrupted() ) return;
380 if( fullMap.containsValue( entry.getValue() ) ) {
381 hitList.add( entry.getKey() );
382 }
383 }
384 }
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421 private void syncMaps( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap, ArrayList<Map<Path, FileAttributes>> resultValue, Path startDestPath,
422 Map<Path, FileAttributes> syncMap ) {
423
424 final Map<Path, FileAttributes> copySourceHitList = resultValue.get( 0 );
425 final Map<Path, FileAttributes> copyDestHitList = resultValue.get( 1 );
426 final Map<Path, FileAttributes> delHitList = resultValue.get( 2 );
427
428 for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
429 if( Thread.currentThread().isInterrupted() ) return;
430
431 final Path relativePath = entry.getValue().getRelativeFilePath();
432 final Path destPath = startDestPath.resolve( relativePath );
433
434 final FileAttributes sourceAttributes = entry.getValue();
435 final FileAttributes destAttributes = destMap.get( destPath );
436 final FileAttributes syncAttributes = syncMap.get( relativePath );
437
438
439
440
441
442
443
444 if( destAttributes == null ) {
445 if( syncAttributes == null ) {
446
447 copySourceHitList.put( entry.getKey(), sourceAttributes );
448 }else {
449
450 delHitList.put( entry.getKey(), sourceAttributes );
451 }
452 continue;
453 }
454
455
456
457
458
459
460
461 if( sourceAttributes.equals( destAttributes ) ) {
462 continue;
463 }
464
465
466
467
468
469
470
471
472
473
474 if( isNewer( sourceAttributes, destAttributes ) ) {
475 copySourceHitList.put( entry.getKey(), sourceAttributes );
476 }else {
477 copyDestHitList.put( destPath, destAttributes );
478 }
479 }
480 }
481
482
483
484
485
486
487
488
489 private boolean isNewer( FileAttributes source, FileAttributes dest ) {
490
491 return source.getModTime().toMillis() > dest.getModTime().toMillis();
492 }
493
494
495
496
497
498
499
500
501
502
503
504
505
506 private Map<Integer, Map<Path, FileAttributes>> splitMap( Map<Path, FileAttributes> map, int avProc ) {
507 final Map<Integer, Map<Path, FileAttributes>> splitedMaps = Model.createMap();
508 for( int i = 0; i < avProc; i++ ) {
509 splitedMaps.put( i, Model.createMap() );
510 }
511 final int split = ( map.size() / avProc ) + 20;
512 int i = 0;
513 int j = 0;
514 for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
515 if( i <= split ) {
516 splitedMaps.get( j ).put( entry.getKey(), entry.getValue() );
517 i++;
518 }else {
519 ++j;
520 i = 0;
521 splitedMaps.get( j ).put( entry.getKey(), entry.getValue() );
522 }
523 }
524 return splitedMaps;
525 }
526 }