View Javadoc
1   package de.spiritscorp.datasync.model;
2   
3   /*-
4    * 		Data Sync
5    *
6    * 		Copyright ©   2022    The Spirit
7    * 		@email                        thespirit@spiritscorp.network
8    *
9    * 		This program is free software; you can redistribute it and/or modify
10   * 		it under the terms of the GNU General Public License as published by
11   * 		the Free Software Foundation; either version 3 of the License, or
12   * 		(at your option) any later version.
13   *
14   * 		This program is distributed in the hope that it will be useful,
15   * 		but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * 		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
17   * 		See the GNU General Public License for more details.
18   *
19   * 		You should have received a copy of the GNU General Public License
20   * 		along with this program. If not, see <http://www.gnu.org/licenses/>.
21   */
22  
23  import java.nio.file.Path;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.HashMap;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.concurrent.ExecutorService;
32  import java.util.concurrent.Executors;
33  import java.util.concurrent.TimeUnit;
34  
35  import de.spiritscorp.datasync.io.Debug;
36  
37  /**
38   * Component responsible for high-level file analysis, conflict detection,
39   * and synchronization state logic.
40   */
41  class FileAnalyzer {
42  
43  	/** The calculated optimal number of parallel threads to use for heavy I/O operations. */
44  	private final int avgProc;
45  	/** The threshold size at which a file list workload is split into sub-tasks for parallel processing. */
46  	private static final int THREAD_SPLIT_SIZE = 30_000;
47  
48  	/**
49  	 * Constructs a new FileAnalyzer and dynamically calculates the optimal thread pool size
50  	 * based on the available system CPU cores.
51  	 */
52  	FileAnalyzer() {
53  		// Allocates roughly half of the available cores minus one if the system has more than 3 cores,
54  		// ensuring the application remains responsive during intense background scanning.
55  		avgProc = ( Runtime.getRuntime().availableProcessors() > 3 ) ? ( Runtime.getRuntime().availableProcessors() / 2 ) - 1 : 1;
56  	}
57  
58  	/**
59  	 * Searches the given map and returns a new map with the duplicates
60  	 *
61  	 * @param sourceMap The map to be checked
62  	 * @return <b>Map</b> <br>
63  	 *         The map with sorted duplicates
64  	 */
65  	Map<Path, FileAttributes> findDuplicates( final Map<Path, FileAttributes> sourceMap ) {
66  		Debug.printDebug( "[File Analyzer]  entryPaths -> %d", sourceMap.size() );
67  		final Map<Path, FileAttributes> duplicateMap = Model.createMap();
68  
69  		final Map<Long, ArrayList<Path>> mapSize = new HashMap<>();
70  		for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
71  			if( Thread.currentThread().isInterrupted() ) return duplicateMap;
72  			final long size = entry.getValue().getSize();
73  			if( mapSize.containsKey( size ) ) {
74  				mapSize.get( size ).add( entry.getKey() );
75  			}else {
76  				mapSize.put( size, new ArrayList<>() );
77  				mapSize.get( size ).add( entry.getKey() );
78  			}
79  		}
80  
81  		for( final Map.Entry<Long, ArrayList<Path>> entry : mapSize.entrySet() ) {
82  			if( Thread.currentThread().isInterrupted() ) return duplicateMap;
83  			final ArrayList<Path> paths = entry.getValue();
84  			if( paths.size() > 1 ) {
85  				for( int i = 0; i < paths.size(); i++ ) {
86  					final String firstPath = sourceMap.get( paths.get( i ) ).getFileHash();
87  					for( int j = i + 1; j < paths.size(); j++ ) {
88  						if( firstPath.equals( sourceMap.get( paths.get( j ) ).getFileHash() ) ) {
89  							duplicateMap.put( paths.get( i ), sourceMap.get( paths.get( i ) ) );
90  							duplicateMap.put( paths.get( j ), sourceMap.get( paths.get( j ) ) );
91  						}
92  					}
93  				}
94  			}
95  		}
96  		Debug.printDebug( "[File Analyzer] DuplicateList -> ready : size: %d", duplicateMap.size() );
97  		return duplicateMap;
98  	}
99  
100 	/**
101 	 * Cleans the maps of identical hits
102 	 *
103 	 * @param sourceMap
104 	 * @param destMap
105 	 */
106 	void equalsFiles( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap ) {
107 		Debug.printDebug( "[File Analyzer] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
108 		if( sourceMap.size() > 0 && destMap.size() > 0 ) {
109 			final Set<Path> sourceHitList = Collections.synchronizedSet( new HashSet<>() );
110 			final Set<Path> destHitList = Collections.synchronizedSet( new HashSet<>() );
111 			if( sourceMap.size() > THREAD_SPLIT_SIZE ) {
112 				try( ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 ) ) {
113 					final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
114 					final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
115 					for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
116 						executor.execute( () -> equalsMap( source.getValue(), destMap, sourceHitList ) );
117 					}
118 					for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
119 						executor.execute( () -> equalsMap( dest.getValue(), sourceMap, destHitList ) );
120 					}
121 					executor.shutdown();
122 					while( !executor.awaitTermination( 10, TimeUnit.MINUTES ) ) {
123 						if( Thread.currentThread().isInterrupted() ) {
124 							executor.shutdownNow();
125 							return;
126 						}
127 					}
128 				}catch( InterruptedException _ ) {
129 					Thread.currentThread().interrupt();
130 					return;
131 				}
132 			}else {
133 				final Thread thread1 = new Thread( () -> equalsMap( sourceMap, destMap, sourceHitList ) );
134 				final Thread thread2 = new Thread( () -> equalsMap( destMap, sourceMap, destHitList ) );
135 				thread1.start();
136 				thread2.start();
137 				try {
138 					thread1.join();
139 					thread2.join();
140 				}catch( InterruptedException _ ) {
141 					thread1.interrupt();
142 					thread2.interrupt();
143 					Thread.currentThread().interrupt();
144 					return;
145 				}
146 			}
147 			for( final Path p : sourceHitList ) {
148 				sourceMap.remove( p );
149 			}
150 			for( final Path p : destHitList ) {
151 				destMap.remove( p );
152 			}
153 			Debug.printDebug( "[File Analyzer] Full source hitList size: %d  && Full destination hitList size: %d", sourceMap.size(), destMap.size() );
154 		}
155 	}
156 
157 	/**
158 	 * Find out which file is the newest version or must be deleted and return the result.
159 	 * Determines synchronization actions by comparing file modification timestamps.
160 	 * <p>
161 	 *
162 	 * @param sourceMap       Map of files from source directory
163 	 * @param destMap         Map of files from destination directory
164 	 * @param startSourcePath Root path of source directory
165 	 * @param startDestPath   Root path of destination directory
166 	 * @param syncMap         Map containing last known synchronization state
167 	 * @return ArrayList containing three maps: [copySource, copyDest, delete]
168 	 */
169 	ArrayList<Map<Path, FileAttributes>> getSyncFiles( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap, final Path startSourcePath, final Path startDestPath,
170 			final Map<Path, FileAttributes> syncMap ) {
171 		Debug.printDebug( "[FileAnalyzer] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
172 		final ArrayList<Map<Path, FileAttributes>> resultValue = new ArrayList<>();
173 		final ArrayList<Map<Path, FileAttributes>> destValue = new ArrayList<>();
174 		final Map<Path, FileAttributes> copySourceHitList = Model.createMap();
175 		final Map<Path, FileAttributes> copyDestHitList = Model.createMap();
176 		final Map<Path, FileAttributes> delHitList = Model.createMap();
177 
178 		resultValue.add( copySourceHitList );
179 		resultValue.add( copyDestHitList );
180 		resultValue.add( delHitList );
181 		destValue.add( copyDestHitList );
182 		destValue.add( copySourceHitList );
183 		destValue.add( delHitList );
184 		if( sourceMap.size() > 0 || destMap.size() > 0 ) {
185 			if( sourceMap.size() > THREAD_SPLIT_SIZE || destMap.size() > THREAD_SPLIT_SIZE ) {
186 				try( ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 ) ) {
187 					final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
188 					final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
189 
190 					for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
191 						executor.execute( () -> syncMaps( source.getValue(), destMap, resultValue, startDestPath, syncMap ) );
192 					}
193 					for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
194 						executor.execute( () -> syncMaps( dest.getValue(), sourceMap, destValue, startSourcePath, syncMap ) );
195 					}
196 					executor.shutdown();
197 					while( !executor.awaitTermination( 10, TimeUnit.MINUTES ) ) {
198 						if( Thread.currentThread().isInterrupted() ) {
199 							executor.shutdownNow();
200 							return resultValue;
201 						}
202 					}
203 				}catch( InterruptedException _ ) {
204 					Thread.currentThread().interrupt();
205 					return resultValue;
206 				}
207 			}else {
208 				syncMaps( sourceMap, destMap, resultValue, startDestPath, syncMap );
209 				if( Thread.currentThread().isInterrupted() ) return resultValue;
210 				syncMaps( destMap, sourceMap, destValue, startSourcePath, syncMap );
211 			}
212 		}
213 		Debug.printDebug( "[File Analyzer] Full copySourceHitList size: %d  && Full copyDestHitList size: %d  && Full delHitList size: %d",
214 				copySourceHitList.size(), copyDestHitList.size(), delHitList.size() );
215 		return resultValue;
216 	}
217 
218 	/**
219 	 * Compares files in the iterate map with files in the full map and records matches.
220 	 * Files are considered equal if their FileAttributes objects are equal
221 	 * (same hash, size, modification time, and name).
222 	 *
223 	 * @param iterateMap The map to iterate through (typically a split/partial map)
224 	 * @param fullMap    The complete map to compare against
225 	 * @param hitList    Set to accumulate matching file paths
226 	 */
227 	private void equalsMap( final Map<Path, FileAttributes> iterateMap, final Map<Path, FileAttributes> fullMap, final Set<Path> hitList ) {
228 		for( final Map.Entry<Path, FileAttributes> entry : iterateMap.entrySet() ) {
229 			if( Thread.currentThread().isInterrupted() ) return;
230 			if( fullMap.containsValue( entry.getValue() ) ) {
231 				hitList.add( entry.getKey() );
232 			}
233 		}
234 	}
235 
236 	/**
237 	 * Determines synchronization actions for all files contained in sourceMap.<br>
238 	 * The synchronization decision is based on:<br>
239 	 * <ul>
240 	 * <li>Current source file state</li>
241 	 * <li>Current destination file state</li>
242 	 * <li>Last known synchronization state</li>
243 	 * <ul><br>
244 	 * Rules:<br>
245 	 * <ol>
246 	 * <li>File only exists in source -> copy to destination</li>
247 	 * <li>File existed previously but is missing in destination -> delete source</li>
248 	 * <li>File exists in source and destination but not in sync state -> initial sync conflict, newest version wins</li>
249 	 * <li>File exists in all locations and differs from sync state -> newest version wins</li>
250 	 * <li>Identical files -> no action</li>
251 	 * <ol>
252 	 *
253 	 * @param sourceMap     Current source files
254 	 * @param destMap       Current destination files
255 	 * @param resultValue   Synchronization result:
256 	 *                      [0] copy source -> destination
257 	 *                      [1] copy destination -> source
258 	 *                      [2] delete files
259 	 * @param startDestPath Destination root path
260 	 * @param syncMap       Last synchronization snapshot
261 	 */
262 	private void syncMaps( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap, final List<Map<Path, FileAttributes>> resultValue, final Path startDestPath,
263 			final Map<Path, FileAttributes> syncMap ) {
264 
265 		final Map<Path, FileAttributes> copySourceHitList = resultValue.get( 0 );
266 		final Map<Path, FileAttributes> copyDestHitList = resultValue.get( 1 );
267 		final Map<Path, FileAttributes> delHitList = resultValue.get( 2 );
268 
269 		for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
270 			if( Thread.currentThread().isInterrupted() ) return;
271 
272 			final Path relativePath = entry.getValue().getRelativeFilePath();
273 			final Path destPath = startDestPath.resolve( relativePath );
274 
275 			final FileAttributes sourceAttributes = entry.getValue();
276 			final FileAttributes destAttributes = destMap.get( destPath );
277 			final FileAttributes syncAttributes = syncMap.get( relativePath );
278 
279 			/*
280 			 * -----------------------------------------------------------------
281 			 * CASE 1
282 			 * File exists only in source.
283 			 * -----------------------------------------------------------------
284 			 */
285 			if( destAttributes == null ) {
286 				if( syncAttributes == null ) {
287 					// New file
288 					copySourceHitList.put( entry.getKey(), sourceAttributes );
289 				}else {
290 					// File existed before but was deleted on destination
291 					delHitList.put( entry.getKey(), sourceAttributes );
292 				}
293 				continue;
294 			}
295 
296 			/*
297 			 * -----------------------------------------------------------------
298 			 * CASE 2
299 			 * File exists in source and destination.
300 			 * -----------------------------------------------------------------
301 			 */
302 			if( sourceAttributes.equals( destAttributes ) ) {
303 				continue;
304 			}
305 
306 			/*
307 			 * -----------------------------------------------------------------
308 			 * CASE 3
309 			 * Initial synchronization conflict.
310 			 *
311 			 * File exists on both sides but there is no sync history.
312 			 * Newest file wins.
313 			 * -----------------------------------------------------------------
314 			 */
315 			if( isNewer( sourceAttributes, destAttributes ) ) {
316 				copySourceHitList.put( entry.getKey(), sourceAttributes );
317 			}else {
318 				copyDestHitList.put( destPath, destAttributes );
319 			}
320 		}
321 	}
322 
323 	/**
324 	 * Returns true if source is newer than destination.
325 	 *
326 	 * @param source Source file attributes
327 	 * @param dest   Destination file attributes
328 	 * @return true if source modification time is newer
329 	 */
330 	private boolean isNewer( final FileAttributes source, final FileAttributes dest ) {
331 
332 		return source.getModTime().toMillis() > dest.getModTime().toMillis();
333 	}
334 
335 	/**
336 	 * Splits a map into smaller chunks for parallel processing.
337 	 * Used to optimize performance when dealing with large file sets.
338 	 *
339 	 * <p>Maps are split based on the number of available processors.
340 	 * Each chunk receives approximately map.{@code size() / avgProc} entries.
341 	 * <p>
342 	 *
343 	 * @param map    The map to split
344 	 * @param avProc Number of threads/chunks to create
345 	 * @return Map of split maps indexed by integer keys (0 to avProc-1)
346 	 */
347 	private Map<Integer, Map<Path, FileAttributes>> splitMap( final Map<Path, FileAttributes> map, final int avProc ) {
348 		final Map<Integer, Map<Path, FileAttributes>> splitedMaps = Model.createMap();
349 		for( int i = 0; i < avProc; i++ ) {
350 			splitedMaps.put( i, Model.createMap() );
351 		}
352 		final int split = ( map.size() / avProc ) + 20;
353 		int innerMap = 0;
354 		int outerMap = 0;
355 		for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
356 			if( innerMap <= split ) {
357 				splitedMaps.get( outerMap ).put( entry.getKey(), entry.getValue() );
358 				innerMap++;
359 			}else {
360 				++outerMap;
361 				innerMap = 0;
362 				splitedMaps.get( outerMap ).put( entry.getKey(), entry.getValue() );
363 			}
364 		}
365 		return splitedMaps;
366 	}
367 }