View Javadoc
1   package de.spiritscorp.datasync.model;
2   
3   /*-
4    * 		Data Sync
5    *
6    * 		Copyright ©   2022    The Spirit
7    * 		@email                        thespirit@spiritscorp.network
8    *
9    * 		This program is free software; you can redistribute it and/or modify
10   * 		it under the terms of the GNU General Public License as published by
11   * 		the Free Software Foundation; either version 3 of the License, or
12   * 		(at your option) any later version.
13   *
14   * 		This program is distributed in the hope that it will be useful,
15   * 		but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * 		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
17   * 		See the GNU General Public License for more details.
18   *
19   * 		You should have received a copy of the GNU General Public License
20   * 		along with this program. If not, see <http://www.gnu.org/licenses/>.
21   */
22  
23  import java.io.IOException;
24  import java.nio.file.Files;
25  import java.nio.file.Path;
26  import java.nio.file.StandardCopyOption;
27  import java.util.ArrayList;
28  import java.util.Collections;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.Map;
32  import java.util.Set;
33  import java.util.concurrent.ExecutorService;
34  import java.util.concurrent.Executors;
35  import java.util.concurrent.TimeUnit;
36  
37  import de.spiritscorp.datasync.ScanType;
38  import de.spiritscorp.datasync.io.Debug;
39  import de.spiritscorp.datasync.io.Logger;
40  
41  class FileHandler {
42  
43  	private final Logger log;
44  	private final int avgProc; // number of threads
45  
46  	FileHandler( Logger log ) {
47  		this.log = log;
48  		avgProc = ( Runtime.getRuntime().availableProcessors() > 3 ) ? ( Runtime.getRuntime().availableProcessors() / 2 ) - 1 : 1;
49  	}
50  
51  	/**
52  	 * List all files in the given directory and execute the filescan for attributes, and give back the results in a new Map
53  	 *
54  	 * @param paths
55  	 * @param deepScan
56  	 * @param subDir
57  	 * @return <b>Map</b> <br>
58  	 *         A map with FileAttributes
59  	 */
60  	void listFiles( ArrayList<Path> paths, Map<Path, FileAttributes> resultMap, ScanType deepScan, boolean subDir ) {
61  		final ExecutorService executor = Executors.newSingleThreadExecutor();
62  		for( final Path path : paths ) {
63  			// Guard: Check interruption context before entering the file tree walker system
64  			if( Thread.currentThread().isInterrupted() ) {
65  				Debug.printDebug( "[FileHandler] Interruption detected prior to walking directory path: %s", path.toString() );
66  				executor.shutdownNow();
67  				return;
68  			}
69  			if( Files.exists( path ) ) {
70  				try {
71  					final Path baseDir = subDir ? path.getParent() : path;
72  					Files.walkFileTree( path, new FileVisit( executor, baseDir, resultMap, deepScan ) );
73  				}catch( final IOException e ) {
74  					Debug.printException( this.getClass(), e );
75  				}
76  			}
77  		}
78  		executor.shutdown();
79  		try {
80  			while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
81  				if( Thread.currentThread().isInterrupted() ) {
82  					executor.shutdownNow();
83  					throw new InterruptedException();
84  				}
85  			}
86  		}catch( final InterruptedException e ) {
87  			Debug.printDebug( "[FileHandler] File processing walk subsystem was forcefully interrupted." );
88  			Thread.currentThread().interrupt();
89  		}
90  		for( final Path path : paths ) {
91  			Debug.printDebug( "[FileHandler] ListFiles() -> ready  %s -> %s", Thread.currentThread().getName(), path.toString() );
92  		}
93  	}
94  
95  	/**
96  	 * Searches the given map and returns a new map with the duplicates
97  	 *
98  	 * @param sourceMap The map to be checked
99  	 * @return <b>Map</b> <br>
100 	 *         The map with sorted duplicates
101 	 */
102 	Map<Path, FileAttributes> findDuplicates( Map<Path, FileAttributes> sourceMap ) {
103 		Debug.printDebug( "[FileHandler]  entryPaths -> %d", sourceMap.size() );
104 		final Map<Path, FileAttributes> duplicateMap = Model.createMap();
105 
106 		final Map<Long, ArrayList<Path>> mapSize = new HashMap<>();
107 		for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
108 			if( Thread.currentThread().isInterrupted() ) return duplicateMap;
109 			final long size = entry.getValue().getSize();
110 			if( mapSize.containsKey( size ) ) {
111 				mapSize.get( size ).add( entry.getKey() );
112 			}else {
113 				mapSize.put( size, new ArrayList<>() );
114 				mapSize.get( size ).add( entry.getKey() );
115 			}
116 		}
117 
118 		for( final Map.Entry<Long, ArrayList<Path>> entry : mapSize.entrySet() ) {
119 			if( Thread.currentThread().isInterrupted() ) return duplicateMap;
120 			final ArrayList<Path> paths = entry.getValue();
121 			if( paths.size() > 1 ) {
122 				for( int i = 0; i < paths.size(); i++ ) {
123 					final String firstPath = sourceMap.get( paths.get( i ) ).getFileHash();
124 					for( int j = i + 1; j < paths.size(); j++ ) {
125 						if( firstPath.equals( sourceMap.get( paths.get( j ) ).getFileHash() ) ) {
126 							duplicateMap.put( paths.get( i ), sourceMap.get( paths.get( i ) ) );
127 							duplicateMap.put( paths.get( j ), sourceMap.get( paths.get( j ) ) );
128 						}
129 					}
130 				}
131 			}
132 		}
133 		Debug.printDebug( "[FileHandler] DuplicateList -> ready : size: %d", duplicateMap.size() );
134 		return duplicateMap;
135 	}
136 
137 	/**
138 	 * Cleans the maps of identical hits
139 	 *
140 	 * @param sourceMap
141 	 * @param destMap
142 	 */
143 	void equalsFiles( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap ) {
144 		Debug.printDebug( "[FileHandler] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
145 		if( sourceMap.size() != 0 && destMap.size() != 0 ) {
146 			final Set<Path> sourceHitList = Collections.synchronizedSet( new HashSet<>() );
147 			final Set<Path> destHitList = Collections.synchronizedSet( new HashSet<>() );
148 			if( sourceMap.size() > 30_000 ) {
149 				final ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 );
150 				final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
151 				final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
152 				for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
153 					executor.execute( () -> equalsMap( source.getValue(), destMap, sourceHitList ) );
154 				}
155 				for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
156 					executor.execute( () -> equalsMap( dest.getValue(), sourceMap, destHitList ) );
157 				}
158 				executor.shutdown();
159 				try {
160 					while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
161 						if( Thread.currentThread().isInterrupted() ) {
162 							executor.shutdownNow();
163 							return;
164 						}
165 					}
166 				}catch( final InterruptedException e ) {
167 					executor.shutdownNow();
168 					Thread.currentThread().interrupt();
169 					return;
170 				}
171 			}else {
172 				final Thread t1 = new Thread( () -> equalsMap( sourceMap, destMap, sourceHitList ) );
173 				final Thread t2 = new Thread( () -> equalsMap( destMap, sourceMap, destHitList ) );
174 				t1.start();
175 				t2.start();
176 				try {
177 					t1.join();
178 					t2.join();
179 				}catch( final InterruptedException e ) {
180 					t1.interrupt();
181 					t2.interrupt();
182 					Thread.currentThread().interrupt();
183 					return;
184 				}
185 			}
186 			// Post-processing guard
187 			if( Thread.currentThread().isInterrupted() ) return;
188 			for( final Path p : sourceHitList ) {
189 				sourceMap.remove( p );
190 			}
191 			for( final Path p : destHitList ) {
192 				destMap.remove( p );
193 			}
194 			Debug.printDebug( "[FileHandler] Full source hitList size: %d  && Full destination hitList size: %d", sourceMap.size(), destMap.size() );
195 		}
196 	}
197 
198 	/**
199 	 * Find out which file is the newest version or must be deleted and return the result.
200 	 * Determines synchronization actions by comparing file modification timestamps.
201 	 * <p>
202 	 *
203 	 * @param sourceMap       Map of files from source directory
204 	 * @param destMap         Map of files from destination directory
205 	 * @param startSourcePath Root path of source directory
206 	 * @param startDestPath   Root path of destination directory
207 	 * @param syncMap         Map containing last known synchronization state
208 	 * @return ArrayList containing three maps: [copySource, copyDest, delete]
209 	 */
210 	ArrayList<Map<Path, FileAttributes>> getSyncFiles( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap, Path startSourcePath, Path startDestPath,
211 			Map<Path, FileAttributes> syncMap ) {
212 		Debug.printDebug( "[FileHandler] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
213 		final ArrayList<Map<Path, FileAttributes>> resultValue = new ArrayList<>();
214 		final ArrayList<Map<Path, FileAttributes>> destValue = new ArrayList<>();
215 		final Map<Path, FileAttributes> copySourceHitList = Model.createMap();
216 		final Map<Path, FileAttributes> copyDestHitList = Model.createMap();
217 		final Map<Path, FileAttributes> delHitList = Model.createMap();
218 
219 		resultValue.add( copySourceHitList );
220 		resultValue.add( copyDestHitList );
221 		resultValue.add( delHitList );
222 		destValue.add( copyDestHitList );
223 		destValue.add( copySourceHitList );
224 		destValue.add( delHitList );
225 		if( sourceMap.size() > 0 || destMap.size() > 0 ) {
226 			if( sourceMap.size() > 30_000 || destMap.size() > 30_000 ) {
227 				final ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 );
228 				final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
229 				final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
230 
231 				for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
232 					executor.execute( () -> syncMaps( source.getValue(), destMap, resultValue, startDestPath, syncMap ) );
233 				}
234 				for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
235 					executor.execute( () -> syncMaps( dest.getValue(), sourceMap, destValue, startSourcePath, syncMap ) );
236 				}
237 				executor.shutdown();
238 				try {
239 					while( !executor.awaitTermination( 100, TimeUnit.MILLISECONDS ) ) {
240 						if( Thread.currentThread().isInterrupted() ) {
241 							executor.shutdownNow();
242 							return resultValue;
243 						}
244 					}
245 				}catch( final InterruptedException e ) {
246 					executor.shutdownNow();
247 					Thread.currentThread().interrupt();
248 					return resultValue;
249 				}
250 			}else {
251 				syncMaps( sourceMap, destMap, resultValue, startDestPath, syncMap );
252 				if( Thread.currentThread().isInterrupted() ) return resultValue;
253 				syncMaps( destMap, sourceMap, destValue, startSourcePath, syncMap );
254 			}
255 		}
256 		Debug.printDebug( "[FileHandler] Full copySourceHitList size: %d  && Full copyDestHitList size: %d  && Full delHitList size: %d",
257 				copySourceHitList.size(), copyDestHitList.size(), delHitList.size() );
258 		return resultValue;
259 	}
260 
261 	/**
262 	 * Deletes files from the specified map with optional trashbin backup.
263 	 *
264 	 * <p>For each file in the map:
265 	 * <ol>
266 	 * <li>Optionally copies file to trashbin directory before deletion</li>
267 	 * <li>Sets write permission if needed</li>
268 	 * <li>Deletes the file</li>
269 	 * <li>Logs the operation result</li>
270 	 * <ol>
271 	 * <p>
272 	 *
273 	 * @param map          Map of files to delete
274 	 * @param logOn        If true, prints status after completion
275 	 * @param trashbin     If true, copies files to trashbin before deletion
276 	 * @param trashbinPath Path to trashbin directory
277 	 */
278 	void deleteFiles( Map<Path, FileAttributes> map, boolean logOn, boolean trashbin, Path trashbinPath ) {
279 		for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
280 			// Guard: Check thread interrupt status before executing file operations
281 			if( Thread.currentThread().isInterrupted() ) {
282 				Debug.printDebug( "[FileHandler] Safe loop interruption caught within file deletion loop vector." );
283 				break;
284 			}
285 
286 			final FileAttributes fileAttr = entry.getValue();
287 			final Path path = entry.getKey();
288 			if( fileAttr == null ) continue;
289 			try {
290 				if( trashbin && trashbinPath != null ) {
291 					Files.createDirectories( trashbinPath.resolve( fileAttr.getRelativeFilePath() ) );
292 					Files.copy( path, trashbinPath.resolve( fileAttr.getRelativeFilePath() ), StandardCopyOption.REPLACE_EXISTING );
293 				}
294 				boolean writable = true;
295 				if( !path.toFile().canWrite() ) writable = path.toFile().setWritable( true );
296 				if( writable ) {
297 					Files.delete( path );
298 					log.setEntry( path.toString(), "gelöscht", fileAttr );
299 				}else {
300 					log.setEntry( path.toString(), "SCHREIBSCHUTZ BEIM LÖSCHEN", fileAttr );
301 					Debug.printDebug( "[FileHandler Error] Delete failed, target file is not writable: %s", path.toString() );
302 				}
303 			}catch( final IOException e ) {
304 				log.setEntry( path.toString(), "FEHLER BEIM LÖSCHEN", fileAttr );
305 				Debug.printDebug( "[FileHandler Error] Delete failed: %s", path.toString() );
306 				Debug.printException( this.getClass(), e );
307 			}
308 		}
309 		map.clear();
310 		if( logOn ) log.printStatus();
311 	}
312 
313 	/**
314 	 * Copies files from source to destination preserving file attributes.
315 	 *
316 	 * <p>For each file in the map:
317 	 * <ol>
318 	 * <li>Creates parent directories if needed</li>
319 	 * <li>Sets write permission on existing destination if needed</li>
320 	 * <li>Copies file with REPLACE_EXISTING and COPY_ATTRIBUTES options</li>
321 	 * <li>Restores original creation time</li>
322 	 * <li>Logs the operation result</li>
323 	 * <ol>
324 	 * <p>
325 	 *
326 	 * @param map      Map of files to copy (key=source path, value=file attributes)
327 	 * @param logOn    If true, prints status after completion
328 	 * @param destPath Destination directory path
329 	 */
330 	void copyFiles( Map<Path, FileAttributes> map, boolean logOn, Path destPath ) {
331 		for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
332 			// Guard: Check thread interrupt status before starting next copy transaction step
333 			if( Thread.currentThread().isInterrupted() ) {
334 				Debug.printDebug( "[FileHandler] Safe loop interruption caught within file replication loop vector." );
335 				break;
336 			}
337 			final FileAttributes fileAttr = entry.getValue();
338 			if( fileAttr == null ) continue;
339 			final Path path = destPath.resolve( fileAttr.getRelativeFilePath() );
340 			final Path parentPath = path.getParent();
341 			boolean writable = true;
342 			try {
343 				if( parentPath != null && !Files.exists( parentPath ) )
344 					Files.createDirectories( parentPath );
345 				else if( Files.exists( path ) && !path.toFile().canWrite() ) writable = path.toFile().setWritable( true );
346 				if( writable ) {
347 					Files.copy(
348 							entry.getKey(),
349 							path,
350 							StandardCopyOption.REPLACE_EXISTING,
351 							StandardCopyOption.COPY_ATTRIBUTES );
352 					Files.setAttribute( path, "creationTime", fileAttr.getCreateTime() );
353 					log.setEntry( path.toString(), "kopiert", fileAttr );
354 				}else {
355 					log.setEntry( path.toString(), "SCHREIBSCHUTZ BEIM KOPIEREN", fileAttr );
356 					Debug.printDebug( "[FileHandler Error] Copy failed, target file is not writable: %s", path.toString() );
357 				}
358 			}catch( final IOException e ) {
359 				log.setEntry( path.toString(), "FEHLER BEIM KOPIEREN", fileAttr );
360 				Debug.printDebug( "[FileHandler Error] Copy failed: %s", path.toString() );
361 				Debug.printException( this.getClass(), e );
362 			}
363 		}
364 		map.clear();
365 		if( logOn ) log.printStatus();
366 	}
367 
368 	/**
369 	 * Compares files in the iterate map with files in the full map and records matches.
370 	 * Files are considered equal if their FileAttributes objects are equal
371 	 * (same hash, size, modification time, and name).
372 	 *
373 	 * @param iterateMap The map to iterate through (typically a split/partial map)
374 	 * @param fullMap    The complete map to compare against
375 	 * @param hitList    Set to accumulate matching file paths
376 	 */
377 	private void equalsMap( Map<Path, FileAttributes> iterateMap, Map<Path, FileAttributes> fullMap, Set<Path> hitList ) {
378 		for( final Map.Entry<Path, FileAttributes> entry : iterateMap.entrySet() ) {
379 			if( Thread.currentThread().isInterrupted() ) return;
380 			if( fullMap.containsValue( entry.getValue() ) ) {
381 				hitList.add( entry.getKey() );
382 			}
383 		}
384 	}
385 
386 	/**
387 	 * Determines synchronization actions for all files contained in sourceMap.
388 	 *
389 	 * <p>
390 	 * The synchronization decision is based on:
391 	 * <p>
392 	 *
393 	 * <ul>
394 	 * <li>Current source file state</li>
395 	 * <li>Current destination file state</li>
396 	 * <li>Last known synchronization state</li>
397 	 * <ul>
398 	 *
399 	 * <p>
400 	 * Rules:
401 	 * <p>
402 	 *
403 	 * <ol>
404 	 * <li>File only exists in source -> copy to destination</li>
405 	 * <li>File existed previously but is missing in destination -> delete source</li>
406 	 * <li>File exists in source and destination but not in sync state
407 	 * -> initial sync conflict, newest version wins</li>
408 	 * <li>File exists in all locations and differs from sync state -> newest version wins</li>
409 	 * <li>Identical files -> no action</li>
410 	 * <ol>
411 	 *
412 	 * @param sourceMap     Current source files
413 	 * @param destMap       Current destination files
414 	 * @param resultValue   Synchronization result:
415 	 *                      [0] copy source -> destination
416 	 *                      [1] copy destination -> source
417 	 *                      [2] delete files
418 	 * @param startDestPath Destination root path
419 	 * @param syncMap       Last synchronization snapshot
420 	 */
421 	private void syncMaps( Map<Path, FileAttributes> sourceMap, Map<Path, FileAttributes> destMap, ArrayList<Map<Path, FileAttributes>> resultValue, Path startDestPath,
422 			Map<Path, FileAttributes> syncMap ) {
423 
424 		final Map<Path, FileAttributes> copySourceHitList = resultValue.get( 0 );
425 		final Map<Path, FileAttributes> copyDestHitList = resultValue.get( 1 );
426 		final Map<Path, FileAttributes> delHitList = resultValue.get( 2 );
427 
428 		for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
429 			if( Thread.currentThread().isInterrupted() ) return;
430 
431 			final Path relativePath = entry.getValue().getRelativeFilePath();
432 			final Path destPath = startDestPath.resolve( relativePath );
433 
434 			final FileAttributes sourceAttributes = entry.getValue();
435 			final FileAttributes destAttributes = destMap.get( destPath );
436 			final FileAttributes syncAttributes = syncMap.get( relativePath );
437 
438 			/*
439 			 * -----------------------------------------------------------------
440 			 * CASE 1
441 			 * File exists only in source.
442 			 * -----------------------------------------------------------------
443 			 */
444 			if( destAttributes == null ) {
445 				if( syncAttributes == null ) {
446 					// New file
447 					copySourceHitList.put( entry.getKey(), sourceAttributes );
448 				}else {
449 					// File existed before but was deleted on destination
450 					delHitList.put( entry.getKey(), sourceAttributes );
451 				}
452 				continue;
453 			}
454 
455 			/*
456 			 * -----------------------------------------------------------------
457 			 * CASE 2
458 			 * File exists in source and destination.
459 			 * -----------------------------------------------------------------
460 			 */
461 			if( sourceAttributes.equals( destAttributes ) ) {
462 				continue;
463 			}
464 
465 			/*
466 			 * -----------------------------------------------------------------
467 			 * CASE 3
468 			 * Initial synchronization conflict.
469 			 *
470 			 * File exists on both sides but there is no sync history.
471 			 * Newest file wins.
472 			 * -----------------------------------------------------------------
473 			 */
474 			if( isNewer( sourceAttributes, destAttributes ) ) {
475 				copySourceHitList.put( entry.getKey(), sourceAttributes );
476 			}else {
477 				copyDestHitList.put( destPath, destAttributes );
478 			}
479 		}
480 	}
481 
482 	/**
483 	 * Returns true if source is newer than destination.
484 	 *
485 	 * @param source Source file attributes
486 	 * @param dest   Destination file attributes
487 	 * @return true if source modification time is newer
488 	 */
489 	private boolean isNewer( FileAttributes source, FileAttributes dest ) {
490 
491 		return source.getModTime().toMillis() > dest.getModTime().toMillis();
492 	}
493 
494 	/**
495 	 * Splits a map into smaller chunks for parallel processing.
496 	 * Used to optimize performance when dealing with large file sets.
497 	 *
498 	 * <p>Maps are split based on the number of available processors.
499 	 * Each chunk receives approximately map.{@code size() / avgProc} entries.
500 	 * <p>
501 	 *
502 	 * @param map    The map to split
503 	 * @param avProc Number of threads/chunks to create
504 	 * @return Map of split maps indexed by integer keys (0 to avProc-1)
505 	 */
506 	private Map<Integer, Map<Path, FileAttributes>> splitMap( Map<Path, FileAttributes> map, int avProc ) {
507 		final Map<Integer, Map<Path, FileAttributes>> splitedMaps = Model.createMap();
508 		for( int i = 0; i < avProc; i++ ) {
509 			splitedMaps.put( i, Model.createMap() );
510 		}
511 		final int split = ( map.size() / avProc ) + 20;
512 		int i = 0;
513 		int j = 0;
514 		for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
515 			if( i <= split ) {
516 				splitedMaps.get( j ).put( entry.getKey(), entry.getValue() );
517 				i++;
518 			}else {
519 				++j;
520 				i = 0;
521 				splitedMaps.get( j ).put( entry.getKey(), entry.getValue() );
522 			}
523 		}
524 		return splitedMaps;
525 	}
526 }