@Test public void testNextARC() throws Exception { String nextArc = storage.nextARC("10"); assertEquals("",nextArc); storage.addARC(arcFile1); nextArc = storage.nextARC("10"); assertEquals(arcFile1,nextArc); storage.addARC(arcFile2); storage.setARCProperties(arcFile2, "10",ArchonConnector.ARC_STATE.NEW, 1); nextArc = storage.nextARC("10"); assertEquals(arcFile2,nextArc); nextArc = storage.nextARC("10"); assertEquals("",nextArc); String arcFile3Shard10Priority3= "arcfile3"; String arcFile4Shard10Priority2= "arcfile4"; storage.addARC(arcFile3Shard10Priority3); storage.addARC(arcFile4Shard10Priority2); storage.setARCProperties(arcFile3Shard10Priority3, "10", ArchonConnector.ARC_STATE.NEW, 3); storage.setARCProperties(arcFile4Shard10Priority2, "10", ArchonConnector.ARC_STATE.NEW, 2); nextArc = storage.nextARC("10"); assertEquals(arcFile3Shard10Priority3,nextArc); storage.clearCachedArcs(); String arcFile5Shard10Priority3= "aaarcfile5"; storage.addARC(arcFile5Shard10Priority3); storage.setARCProperties(arcFile5Shard10Priority3, "10", ArchonConnector.ARC_STATE.NEW, 3); nextArc = storage.nextARC("10"); assertEquals(arcFile5Shard10Priority3,nextArc); nextArc = storage.nextARC("12"); assertEquals("",nextArc); }
public String nextARC(String shardID) throws Exception{ PreparedStatement stmt = null; try { ArrayList<ArcVO> nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard == null || nextArcIdsCachedForShard.size() == 0){ ArrayList<ArcVO> nextArcIds = getNextArcIdsToCache(shardID); nextArcIdsCached.put(shardID, nextArcIds); log.info("initialized nextArcIdCache for shardID:"+shardID); } nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard.size() == 0){ log.info("No ARC files with status NEW are ready to process."); return ""; } ArcVO nextArc = nextArcIdsCachedForShard.remove(0); Set<String> keySet = nextArcIdsCached.keySet(); for (String shardId: keySet){ boolean removed = nextArcIdsCached.get(shardId).remove(nextArc); if (removed){ log.info("warc was also removed from cache for shardid:"+shardId); } } String fullPath = nextArc.getFileName(); setARCProperties(nextArc.getFileName(), shardID, ArchonConnector.ARC_STATE.RUNNING, nextArc.getPriority()); log.info("Returning next arc:"+fullPath); return fullPath; } catch (SQLException e) { log.error("SQL Exception in nextARC:" + e.getMessage()); throw e; } finally { closeStatement(stmt); } }
ArchonStorage { public String nextARC(String shardID) throws Exception{ PreparedStatement stmt = null; try { ArrayList<ArcVO> nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard == null || nextArcIdsCachedForShard.size() == 0){ ArrayList<ArcVO> nextArcIds = getNextArcIdsToCache(shardID); nextArcIdsCached.put(shardID, nextArcIds); log.info("initialized nextArcIdCache for shardID:"+shardID); } nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard.size() == 0){ log.info("No ARC files with status NEW are ready to process."); return ""; } ArcVO nextArc = nextArcIdsCachedForShard.remove(0); Set<String> keySet = nextArcIdsCached.keySet(); for (String shardId: keySet){ boolean removed = nextArcIdsCached.get(shardId).remove(nextArc); if (removed){ log.info("warc was also removed from cache for shardid:"+shardId); } } String fullPath = nextArc.getFileName(); setARCProperties(nextArc.getFileName(), shardID, ArchonConnector.ARC_STATE.RUNNING, nextArc.getPriority()); log.info("Returning next arc:"+fullPath); return fullPath; } catch (SQLException e) { log.error("SQL Exception in nextARC:" + e.getMessage()); throw e; } finally { closeStatement(stmt); } } }
ArchonStorage { public String nextARC(String shardID) throws Exception{ PreparedStatement stmt = null; try { ArrayList<ArcVO> nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard == null || nextArcIdsCachedForShard.size() == 0){ ArrayList<ArcVO> nextArcIds = getNextArcIdsToCache(shardID); nextArcIdsCached.put(shardID, nextArcIds); log.info("initialized nextArcIdCache for shardID:"+shardID); } nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard.size() == 0){ log.info("No ARC files with status NEW are ready to process."); return ""; } ArcVO nextArc = nextArcIdsCachedForShard.remove(0); Set<String> keySet = nextArcIdsCached.keySet(); for (String shardId: keySet){ boolean removed = nextArcIdsCached.get(shardId).remove(nextArc); if (removed){ log.info("warc was also removed from cache for shardid:"+shardId); } } String fullPath = nextArc.getFileName(); setARCProperties(nextArc.getFileName(), shardID, ArchonConnector.ARC_STATE.RUNNING, nextArc.getPriority()); log.info("Returning next arc:"+fullPath); return fullPath; } catch (SQLException e) { log.error("SQL Exception in nextARC:" + e.getMessage()); throw e; } finally { closeStatement(stmt); } } ArchonStorage(); }
ArchonStorage { public String nextARC(String shardID) throws Exception{ PreparedStatement stmt = null; try { ArrayList<ArcVO> nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard == null || nextArcIdsCachedForShard.size() == 0){ ArrayList<ArcVO> nextArcIds = getNextArcIdsToCache(shardID); nextArcIdsCached.put(shardID, nextArcIds); log.info("initialized nextArcIdCache for shardID:"+shardID); } nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard.size() == 0){ log.info("No ARC files with status NEW are ready to process."); return ""; } ArcVO nextArc = nextArcIdsCachedForShard.remove(0); Set<String> keySet = nextArcIdsCached.keySet(); for (String shardId: keySet){ boolean removed = nextArcIdsCached.get(shardId).remove(nextArc); if (removed){ log.info("warc was also removed from cache for shardid:"+shardId); } } String fullPath = nextArc.getFileName(); setARCProperties(nextArc.getFileName(), shardID, ArchonConnector.ARC_STATE.RUNNING, nextArc.getPriority()); log.info("Returning next arc:"+fullPath); return fullPath; } catch (SQLException e) { log.error("SQL Exception in nextARC:" + e.getMessage()); throw e; } finally { closeStatement(stmt); } } ArchonStorage(); static void initialize(String driverName, String driverUrl, String userName, String password); int nextShardId(); void addARC(String arcPath); void addOrUpdateARC(String arcPath); List<String> getShardIDs(); String nextARC(String shardID); List<String> getARCFiles(String shardID); ArcVO getArcByID(String arcID); int getArcCount(); List<ArcVO> getLatest1000Arcs(); List<ArcVO> getAllRunningArcs(); void setARCState(String arcPath, ArchonConnector.ARC_STATE arcState); void setShardState(String shardID, ArchonConnector.ARC_STATE state, int priority); void setARCProperties(String fullPath, String shardID,ArchonConnector.ARC_STATE state, int priority); void setARCPriority(String fullPath, int priority); void resetArcWithPriorityStatement(String fullPath, int priority); void removeARC(String arcID); void clearIndexing(String shardID); void resetShardId(String shardID); void commit(); void rollback(); void close(); static void shutdown(); }
ArchonStorage { public String nextARC(String shardID) throws Exception{ PreparedStatement stmt = null; try { ArrayList<ArcVO> nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard == null || nextArcIdsCachedForShard.size() == 0){ ArrayList<ArcVO> nextArcIds = getNextArcIdsToCache(shardID); nextArcIdsCached.put(shardID, nextArcIds); log.info("initialized nextArcIdCache for shardID:"+shardID); } nextArcIdsCachedForShard = nextArcIdsCached.get(shardID); if (nextArcIdsCachedForShard.size() == 0){ log.info("No ARC files with status NEW are ready to process."); return ""; } ArcVO nextArc = nextArcIdsCachedForShard.remove(0); Set<String> keySet = nextArcIdsCached.keySet(); for (String shardId: keySet){ boolean removed = nextArcIdsCached.get(shardId).remove(nextArc); if (removed){ log.info("warc was also removed from cache for shardid:"+shardId); } } String fullPath = nextArc.getFileName(); setARCProperties(nextArc.getFileName(), shardID, ArchonConnector.ARC_STATE.RUNNING, nextArc.getPriority()); log.info("Returning next arc:"+fullPath); return fullPath; } catch (SQLException e) { log.error("SQL Exception in nextARC:" + e.getMessage()); throw e; } finally { closeStatement(stmt); } } ArchonStorage(); static void initialize(String driverName, String driverUrl, String userName, String password); int nextShardId(); void addARC(String arcPath); void addOrUpdateARC(String arcPath); List<String> getShardIDs(); String nextARC(String shardID); List<String> getARCFiles(String shardID); ArcVO getArcByID(String arcID); int getArcCount(); List<ArcVO> getLatest1000Arcs(); List<ArcVO> getAllRunningArcs(); void setARCState(String arcPath, ArchonConnector.ARC_STATE arcState); void setShardState(String shardID, ArchonConnector.ARC_STATE state, int priority); void setARCProperties(String fullPath, String shardID,ArchonConnector.ARC_STATE state, int priority); void setARCPriority(String fullPath, int priority); void resetArcWithPriorityStatement(String fullPath, int priority); void removeARC(String arcID); void clearIndexing(String shardID); void resetShardId(String shardID); void commit(); void rollback(); void close(); static void shutdown(); static Date INITDATE; }