1 package de.spiritscorp.datasync.model;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 import java.nio.file.Path;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.concurrent.ExecutorService;
32 import java.util.concurrent.Executors;
33 import java.util.concurrent.TimeUnit;
34
35 import de.spiritscorp.datasync.io.Debug;
36
37
38
39
40
41 class FileAnalyzer {
42
43
44 private final int avgProc;
45
46 private static final int THREAD_SPLIT_SIZE = 30_000;
47
48
49
50
51
52 FileAnalyzer() {
53
54
55 avgProc = ( Runtime.getRuntime().availableProcessors() > 3 ) ? ( Runtime.getRuntime().availableProcessors() / 2 ) - 1 : 1;
56 }
57
58
59
60
61
62
63
64
65 Map<Path, FileAttributes> findDuplicates( final Map<Path, FileAttributes> sourceMap ) {
66 Debug.printDebug( "[File Analyzer] entryPaths -> %d", sourceMap.size() );
67 final Map<Path, FileAttributes> duplicateMap = Model.createMap();
68
69 final Map<Long, ArrayList<Path>> mapSize = new HashMap<>();
70 for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
71 if( Thread.currentThread().isInterrupted() ) return duplicateMap;
72 final long size = entry.getValue().getSize();
73 if( mapSize.containsKey( size ) ) {
74 mapSize.get( size ).add( entry.getKey() );
75 }else {
76 mapSize.put( size, new ArrayList<>() );
77 mapSize.get( size ).add( entry.getKey() );
78 }
79 }
80
81 for( final Map.Entry<Long, ArrayList<Path>> entry : mapSize.entrySet() ) {
82 if( Thread.currentThread().isInterrupted() ) return duplicateMap;
83 final ArrayList<Path> paths = entry.getValue();
84 if( paths.size() > 1 ) {
85 for( int i = 0; i < paths.size(); i++ ) {
86 final String firstPath = sourceMap.get( paths.get( i ) ).getFileHash();
87 for( int j = i + 1; j < paths.size(); j++ ) {
88 if( firstPath.equals( sourceMap.get( paths.get( j ) ).getFileHash() ) ) {
89 duplicateMap.put( paths.get( i ), sourceMap.get( paths.get( i ) ) );
90 duplicateMap.put( paths.get( j ), sourceMap.get( paths.get( j ) ) );
91 }
92 }
93 }
94 }
95 }
96 Debug.printDebug( "[File Analyzer] DuplicateList -> ready : size: %d", duplicateMap.size() );
97 return duplicateMap;
98 }
99
100
101
102
103
104
105
106 void equalsFiles( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap ) {
107 Debug.printDebug( "[File Analyzer] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
108 if( sourceMap.size() > 0 && destMap.size() > 0 ) {
109 final Set<Path> sourceHitList = Collections.synchronizedSet( new HashSet<>() );
110 final Set<Path> destHitList = Collections.synchronizedSet( new HashSet<>() );
111 if( sourceMap.size() > THREAD_SPLIT_SIZE ) {
112 try( ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 ) ) {
113 final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
114 final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
115 for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
116 executor.execute( () -> equalsMap( source.getValue(), destMap, sourceHitList ) );
117 }
118 for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
119 executor.execute( () -> equalsMap( dest.getValue(), sourceMap, destHitList ) );
120 }
121 executor.shutdown();
122 while( !executor.awaitTermination( 10, TimeUnit.MINUTES ) ) {
123 if( Thread.currentThread().isInterrupted() ) {
124 executor.shutdownNow();
125 return;
126 }
127 }
128 }catch( InterruptedException _ ) {
129 Thread.currentThread().interrupt();
130 return;
131 }
132 }else {
133 final Thread thread1 = new Thread( () -> equalsMap( sourceMap, destMap, sourceHitList ) );
134 final Thread thread2 = new Thread( () -> equalsMap( destMap, sourceMap, destHitList ) );
135 thread1.start();
136 thread2.start();
137 try {
138 thread1.join();
139 thread2.join();
140 }catch( InterruptedException _ ) {
141 thread1.interrupt();
142 thread2.interrupt();
143 Thread.currentThread().interrupt();
144 return;
145 }
146 }
147 for( final Path p : sourceHitList ) {
148 sourceMap.remove( p );
149 }
150 for( final Path p : destHitList ) {
151 destMap.remove( p );
152 }
153 Debug.printDebug( "[File Analyzer] Full source hitList size: %d && Full destination hitList size: %d", sourceMap.size(), destMap.size() );
154 }
155 }
156
157
158
159
160
161
162
163
164
165
166
167
168
169 ArrayList<Map<Path, FileAttributes>> getSyncFiles( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap, final Path startSourcePath, final Path startDestPath,
170 final Map<Path, FileAttributes> syncMap ) {
171 Debug.printDebug( "[FileAnalyzer] max mem: %d, free mem: %d, total mem: %d", Runtime.getRuntime().maxMemory(), Runtime.getRuntime().freeMemory(), Runtime.getRuntime().totalMemory() );
172 final ArrayList<Map<Path, FileAttributes>> resultValue = new ArrayList<>();
173 final ArrayList<Map<Path, FileAttributes>> destValue = new ArrayList<>();
174 final Map<Path, FileAttributes> copySourceHitList = Model.createMap();
175 final Map<Path, FileAttributes> copyDestHitList = Model.createMap();
176 final Map<Path, FileAttributes> delHitList = Model.createMap();
177
178 resultValue.add( copySourceHitList );
179 resultValue.add( copyDestHitList );
180 resultValue.add( delHitList );
181 destValue.add( copyDestHitList );
182 destValue.add( copySourceHitList );
183 destValue.add( delHitList );
184 if( sourceMap.size() > 0 || destMap.size() > 0 ) {
185 if( sourceMap.size() > THREAD_SPLIT_SIZE || destMap.size() > THREAD_SPLIT_SIZE ) {
186 try( ExecutorService executor = Executors.newFixedThreadPool( avgProc * 2 ) ) {
187 final Map<Integer, Map<Path, FileAttributes>> splitSource = splitMap( sourceMap, avgProc );
188 final Map<Integer, Map<Path, FileAttributes>> splitDest = splitMap( destMap, avgProc );
189
190 for( final Map.Entry<Integer, Map<Path, FileAttributes>> source : splitSource.entrySet() ) {
191 executor.execute( () -> syncMaps( source.getValue(), destMap, resultValue, startDestPath, syncMap ) );
192 }
193 for( final Map.Entry<Integer, Map<Path, FileAttributes>> dest : splitDest.entrySet() ) {
194 executor.execute( () -> syncMaps( dest.getValue(), sourceMap, destValue, startSourcePath, syncMap ) );
195 }
196 executor.shutdown();
197 while( !executor.awaitTermination( 10, TimeUnit.MINUTES ) ) {
198 if( Thread.currentThread().isInterrupted() ) {
199 executor.shutdownNow();
200 return resultValue;
201 }
202 }
203 }catch( InterruptedException _ ) {
204 Thread.currentThread().interrupt();
205 return resultValue;
206 }
207 }else {
208 syncMaps( sourceMap, destMap, resultValue, startDestPath, syncMap );
209 if( Thread.currentThread().isInterrupted() ) return resultValue;
210 syncMaps( destMap, sourceMap, destValue, startSourcePath, syncMap );
211 }
212 }
213 Debug.printDebug( "[File Analyzer] Full copySourceHitList size: %d && Full copyDestHitList size: %d && Full delHitList size: %d",
214 copySourceHitList.size(), copyDestHitList.size(), delHitList.size() );
215 return resultValue;
216 }
217
218
219
220
221
222
223
224
225
226
227 private void equalsMap( final Map<Path, FileAttributes> iterateMap, final Map<Path, FileAttributes> fullMap, final Set<Path> hitList ) {
228 for( final Map.Entry<Path, FileAttributes> entry : iterateMap.entrySet() ) {
229 if( Thread.currentThread().isInterrupted() ) return;
230 if( fullMap.containsValue( entry.getValue() ) ) {
231 hitList.add( entry.getKey() );
232 }
233 }
234 }
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 private void syncMaps( final Map<Path, FileAttributes> sourceMap, final Map<Path, FileAttributes> destMap, final List<Map<Path, FileAttributes>> resultValue, final Path startDestPath,
263 final Map<Path, FileAttributes> syncMap ) {
264
265 final Map<Path, FileAttributes> copySourceHitList = resultValue.get( 0 );
266 final Map<Path, FileAttributes> copyDestHitList = resultValue.get( 1 );
267 final Map<Path, FileAttributes> delHitList = resultValue.get( 2 );
268
269 for( final Map.Entry<Path, FileAttributes> entry : sourceMap.entrySet() ) {
270 if( Thread.currentThread().isInterrupted() ) return;
271
272 final Path relativePath = entry.getValue().getRelativeFilePath();
273 final Path destPath = startDestPath.resolve( relativePath );
274
275 final FileAttributes sourceAttributes = entry.getValue();
276 final FileAttributes destAttributes = destMap.get( destPath );
277 final FileAttributes syncAttributes = syncMap.get( relativePath );
278
279
280
281
282
283
284
285 if( destAttributes == null ) {
286 if( syncAttributes == null ) {
287
288 copySourceHitList.put( entry.getKey(), sourceAttributes );
289 }else {
290
291 delHitList.put( entry.getKey(), sourceAttributes );
292 }
293 continue;
294 }
295
296
297
298
299
300
301
302 if( sourceAttributes.equals( destAttributes ) ) {
303 continue;
304 }
305
306
307
308
309
310
311
312
313
314
315 if( isNewer( sourceAttributes, destAttributes ) ) {
316 copySourceHitList.put( entry.getKey(), sourceAttributes );
317 }else {
318 copyDestHitList.put( destPath, destAttributes );
319 }
320 }
321 }
322
323
324
325
326
327
328
329
330 private boolean isNewer( final FileAttributes source, final FileAttributes dest ) {
331
332 return source.getModTime().toMillis() > dest.getModTime().toMillis();
333 }
334
335
336
337
338
339
340
341
342
343
344
345
346
347 private Map<Integer, Map<Path, FileAttributes>> splitMap( final Map<Path, FileAttributes> map, final int avProc ) {
348 final Map<Integer, Map<Path, FileAttributes>> splitedMaps = Model.createMap();
349 for( int i = 0; i < avProc; i++ ) {
350 splitedMaps.put( i, Model.createMap() );
351 }
352 final int split = ( map.size() / avProc ) + 20;
353 int innerMap = 0;
354 int outerMap = 0;
355 for( final Map.Entry<Path, FileAttributes> entry : map.entrySet() ) {
356 if( innerMap <= split ) {
357 splitedMaps.get( outerMap ).put( entry.getKey(), entry.getValue() );
358 innerMap++;
359 }else {
360 ++outerMap;
361 innerMap = 0;
362 splitedMaps.get( outerMap ).put( entry.getKey(), entry.getValue() );
363 }
364 }
365 return splitedMaps;
366 }
367 }