ArcDigitalAssetStoreService implements DigitalAssetStore,
        LogProvider { public HarvestResultDTO copyAndPrune(String targetInstanceName, int orgHarvestResultNum, int newHarvestResultNum, List<String> urisToDelete, List<HarvestResourceDTO> hrsToImport) throws DigitalAssetStoreException { try { File sourceDir = new File(baseDir, targetInstanceName + "/" + orgHarvestResultNum); File destDir = new File(baseDir, targetInstanceName + "/" + newHarvestResultNum); destDir.mkdirs(); File[] arcFiles = sourceDir.listFiles(); List<File> dirs = new LinkedList<File>(); dirs.add(destDir); boolean compressed = false; AtomicInteger aint = new AtomicInteger(); String impArcType = null; String strippedImpArcFilename = null; List<String> impArcHeader = new ArrayList<String>(); for (int i = 0; i < arcFiles.length; i++) { if (arcFiles[i].getName().toUpperCase().endsWith("CDX")) { continue; } ArchiveReader reader = ArchiveReaderFactory.get(arcFiles[i]); strippedImpArcFilename = reader.getStrippedFileName(); compressed = reader.isCompressed(); Iterator<ArchiveRecord> archiveRecordsIt = reader.iterator(); if (reader instanceof ARCReader) { if (impArcType == null) { impArcType = "ARC"; } ARCRecord headerRec = (ARCRecord) archiveRecordsIt.next(); byte[] buff = new byte[1024]; StringBuffer metaData = new StringBuffer(); int bytesRead = 0; while ((bytesRead = headerRec.read(buff)) != -1) { metaData.append(new String(buff, 0, bytesRead)); } List<String> l = new ArrayList<String>(); l.add(metaData.toString()); if (impArcHeader.isEmpty()) { impArcHeader.add(metaData.toString()); } WriterPoolSettings settings = new WriterPoolSettingsData(strippedImpArcFilename + "-" + newHarvestResultNum, "${prefix}", ARCReader.DEFAULT_MAX_ARC_FILE_SIZE, compressed, dirs, l); ARCWriter writer = new ARCWriter(aint, settings); while (archiveRecordsIt.hasNext()) { ARCRecord record = (ARCRecord) archiveRecordsIt.next(); ARCRecordMetaData meta = record.getMetaData(); Date dt; try { dt = sdf.parse(meta.getDate()); } catch (ParseException ex) { dt = new Date(); if (log.isWarnEnabled()) { log.warn("Couldn't parse date from ARCRecord: " + record.getMetaData().getUrl(), ex); log.warn("Setting to the current date."); } } if (!urisToDelete.contains(meta.getUrl())) { if (!listContainsURL(hrsToImport, meta.getUrl())) { writer.write(meta.getUrl(), meta.getMimetype(), meta.getIp(), dt.getTime(), (int) meta.getLength(), record); } } } writer.close(); } else if (reader instanceof WARCReader) { if (impArcType == null) { impArcType = "WARC"; } ArchiveReader headerReader = ArchiveReaderFactory.get(arcFiles[i]); Iterator<ArchiveRecord> headerRecordIt = headerReader.iterator(); WARCRecord headerRec = (WARCRecord) headerRecordIt.next(); byte[] buff = new byte[1024]; StringBuffer metaData = new StringBuffer(); int bytesRead = 0; while ((bytesRead = headerRec.read(buff)) != -1) { metaData.append(new String(buff, 0, bytesRead)); } List<String> l = new ArrayList<String>(); l.add(metaData.toString()); if (impArcHeader.isEmpty()) { impArcHeader.add(metaData.toString()); } headerRec.close(); headerReader.close(); archiveRecordsIt.next(); WARCWriterPoolSettings settings = new WARCWriterPoolSettingsData(strippedImpArcFilename + "-" + newHarvestResultNum, "${prefix}", ARCReader.DEFAULT_MAX_ARC_FILE_SIZE, compressed, dirs, l, new UUIDGenerator()); WARCWriter writer = new WARCWriter(aint, settings); while (archiveRecordsIt.hasNext()) { WARCRecord record = (WARCRecord) archiveRecordsIt .next(); ArchiveRecordHeader header = record.getHeader(); String WARCType = (String) header .getHeaderValue(WARCConstants.HEADER_KEY_TYPE); String strRecordId = (String) header .getHeaderValue(WARCConstants.HEADER_KEY_ID); URI recordId = new URI(strRecordId.substring( strRecordId.indexOf("<") + 1, strRecordId.lastIndexOf(">") - 1)); long contentLength = header.getLength() - header.getContentBegin(); if (!WARCType.equals(WARCConstants.WARCRecordType.warcinfo) && (urisToDelete.contains(header.getUrl()) || listContainsURL( hrsToImport, header.getUrl()))) { continue; } ANVLRecord namedFields = new ANVLRecord(); Iterator hdrFieldsIt = header.getHeaderFieldKeys() .iterator(); while (hdrFieldsIt.hasNext()) { String key = (String) hdrFieldsIt.next(); String value = header.getHeaderValue(key) .toString(); if (key.equals(WARCConstants.ABSOLUTE_OFFSET_KEY)) { value = new Long(writer.getPosition()) .toString(); } if (key.equals("WARC-IP-Address") || key.equals("WARC-Payload-Digest") || key.equals("WARC-Concurrent-To")) namedFields.addLabelValue(key, value); } WARCRecordInfo warcRecordInfo = new WARCRecordInfo(); switch (WARCConstants.WARCRecordType.valueOf(WARCType)) { case warcinfo: warcRecordInfo.setType(WARCConstants.WARCRecordType.warcinfo); break; case response: warcRecordInfo.setType(WARCConstants.WARCRecordType.response); warcRecordInfo.setUrl(header.getUrl()); break; case metadata: warcRecordInfo.setType(WARCConstants.WARCRecordType.metadata); warcRecordInfo.setUrl(header.getUrl()); break; case request: warcRecordInfo.setType(WARCConstants.WARCRecordType.request); warcRecordInfo.setUrl(header.getUrl()); break; case resource: warcRecordInfo.setType(WARCConstants.WARCRecordType.resource); warcRecordInfo.setUrl(header.getUrl()); break; case revisit: warcRecordInfo.setType(WARCConstants.WARCRecordType.revisit); warcRecordInfo.setUrl(header.getUrl()); break; default: if (log.isWarnEnabled()) { log.warn("Ignoring unrecognised type for WARCRecord: " + WARCType); } } warcRecordInfo.setCreate14DigitDate(header.getDate()); warcRecordInfo.setMimetype(header.getMimetype()); warcRecordInfo.setRecordId(recordId); warcRecordInfo.setExtraHeaders(namedFields); warcRecordInfo.setContentStream(record); warcRecordInfo.setContentLength(contentLength); writer.writeRecord(warcRecordInfo); } writer.close(); } reader.close(); } if (!hrsToImport.isEmpty()) { if (impArcType.equals("ARC")) { WriterPoolSettings settings = new WriterPoolSettingsData(strippedImpArcFilename + "-new", "${prefix}", ARCReader.DEFAULT_MAX_ARC_FILE_SIZE, compressed, dirs, impArcHeader); ARCWriter arcWriter = new ARCWriter(aint, settings); for (Iterator<HarvestResourceDTO> it = hrsToImport .iterator(); it.hasNext(); ) { HarvestResourceDTO hr = it.next(); if (hr.getLength() > 0L) { File fin = new File(this.baseDir, "/uploadedFiles/" + hr.getTempFileName()); Date dtNow = new Date(); arcWriter.write(hr.getName(), hr.getContentType(), "0.0.0.0", dtNow.getTime(), hr.getLength(), new java.io.FileInputStream(fin)); } } arcWriter.close(); } else { WARCWriterPoolSettings settings = new WARCWriterPoolSettingsData(strippedImpArcFilename +"-new", "${prefix}", WARCReader.DEFAULT_MAX_WARC_FILE_SIZE, compressed, dirs, impArcHeader, new UUIDGenerator()); WARCWriter warcWriter = new WARCWriter(aint, settings); for (Iterator<HarvestResourceDTO> it = hrsToImport .iterator(); it.hasNext(); ) { HarvestResourceDTO hr = it.next(); if (hr.getLength() > 0L) { File fin = new File(this.baseDir, "/uploadedFiles/" + hr.getTempFileName()); Date dtNow = new Date(); URI recordId = new URI("urn:uuid:" + hr.getTempFileName()); ANVLRecord namedFields = new ANVLRecord(); namedFields.addLabelValue( WARCConstants.HEADER_KEY_IP, "0.0.0.0"); WARCRecordInfo warcRecordInfo = new WARCRecordInfo(); warcRecordInfo.setUrl(hr.getName()); warcRecordInfo.setCreate14DigitDate(writerDF.format(dtNow)); warcRecordInfo.setMimetype(hr.getContentType()); warcRecordInfo.setRecordId(recordId); warcRecordInfo.setExtraHeaders(namedFields); warcRecordInfo.setContentStream(new java.io.FileInputStream(fin)); warcRecordInfo.setContentLength(hr.getLength()); warcRecordInfo.setType(WARCConstants.WARCRecordType.response); warcWriter.writeRecord(warcRecordInfo); } } warcWriter.close(); } } log.info("copyAndPrune - Now time to reindex."); ArcHarvestResultDTO ahr = new ArcHarvestResultDTO(); File[] fileList = destDir.listFiles(); Set<ArcHarvestFileDTO> fileset = new HashSet<ArcHarvestFileDTO>(); for (File f : fileList) { ArcHarvestFileDTO ahf = new ArcHarvestFileDTO(); ahf.setCompressed(compressed); ahf.setName(f.getName()); fileset.add(ahf); } ahr.setArcFiles(fileset); ahr.setCreationDate(new Date()); ahr.setHarvestNumber(newHarvestResultNum); ahr.index(destDir); log.info("copyAndPrune - Now returning the ArcHarvestResult: " + ahr.getOid()); return ahr; } catch (URISyntaxException e) { if (log.isErrorEnabled()) { log.error("Prune and Copy Failed : " + e.getMessage(), e); } throw new DigitalAssetStoreException("Prune and Copy Failed : " + e.getMessage(), e); } catch (IOException e) { if (log.isErrorEnabled()) { log.error("Prune and Copy Failed : " + e.getMessage(), e); } throw new DigitalAssetStoreException("Prune and Copy Failed : " + e.getMessage(), e); } catch (ParseException e) { if (log.isErrorEnabled()) { log.error("Prune and Copy Failed : " + e.getMessage(), e); } throw new DigitalAssetStoreException("Prune and Copy Failed : " + e.getMessage(), e); } catch (Exception e) { log.info(e.getMessage()); e.printStackTrace(); throw new DigitalAssetStoreException("Prune and Copy Failed : " + e.getMessage(), e); } }  void save(String targetInstanceName, String directory, File file); void save(String targetInstanceName, String directory, File[] files); void save(String targetInstanceName, File[] files); void save(String targetInstanceName, File file); @SuppressWarnings("finally") File getResource(String targetInstanceName, int harvestResultNumber,
                            HarvestResourceDTO resourcex); byte[] getSmallResource(String targetInstanceName,
                                   int harvestResultNumber, HarvestResourceDTO resourcex); Header[] getHeaders(String targetInstanceName,
                               int harvestResultNumber, HarvestResourceDTO resourcex); HarvestResultDTO copyAndPrune(String targetInstanceName,
                                         int orgHarvestResultNum, int newHarvestResultNum,
                                         List<String> urisToDelete, List<HarvestResourceDTO> hrsToImport); File getLogFile(String aJob, String aFileName); List<String> getLogFileNames(String aJob); LogFilePropertiesDTO[] getLogFileAttributes(String aJob); void purge(String[] targetInstanceNames); void purgeAbortedTargetInstances(String[] targetInstanceNames); void submitToArchive(String targetInstanceOid, String SIP,
                                Map xAttributes, int harvestNumber); CustomDepositFormResultDTO getCustomDepositFormDetails(
            CustomDepositFormCriteriaDTO criteria); void setArchive(Archive archive); void setBaseDir(String baseDir); void initiateIndexing(ArcHarvestResultDTO harvestResult); void initiateRemoveIndexes(ArcHarvestResultDTO harvestResult); Boolean checkIndexing(Long harvestResultOid); void setIndexer(Indexer indexer); void setDasFileMover(DasFileMover fileMover); void setWsEndPoint(WebServiceEndPoint wsEndPoint); void setPageImagePrefix(String pageImagePrefix); String getPageImagePrefix(); void setAqaReportPrefix(String aqaReportPrefix); String getAqaReportPrefix(); void setFileArchive(FileArchive fileArchive);  }
@Ignore @Test public final void testCopyAndPrune() throws Exception { String targetInstanceName = new Long(targetInstanceOid).toString(); String uri = "http: File destDir = new File(baseDir, targetInstanceName + "/" + (harvestResultNumber + 1)); delDir(destDir); List<String> urisToDelete = new ArrayList<String>(); List<HarvestResourceDTO> hrsToImport = new ArrayList<HarvestResourceDTO>(); urisToDelete.add(uri); HarvestResultDTO dto = testInstance.copyAndPrune(targetInstanceName, harvestResultNumber, harvestResultNumber + 1, urisToDelete, hrsToImport); assertNotNull(dto); assertTrue(destDir.exists()); File[] arcFiles = destDir.listFiles(new ARCFilter()); assertEquals(1, arcFiles.length); delDir(destDir); }