@Test public void testRegionServerAbortionDueToFailureTransitioningToOpened() throws IOException, NodeExistsException, KeeperException { final Server server = new MockServer(HTU); final RegionServerServices rss = HTU.createMockRegionServerService(); HTableDescriptor htd = TEST_HTD; final HRegionInfo hri = TEST_HRI; HRegion region = HRegion.createHRegion(hri, HTU.getDataTestDir(), HTU .getConfiguration(), htd); assertNotNull(region); try { ZkCoordinatedStateManager csm = new ZkCoordinatedStateManager(); csm.initialize(server); csm.start(); ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = new ZkOpenRegionCoordination.ZkOpenRegionDetails(); zkCrd.setServerName(server.getServerName()); ZkOpenRegionCoordination openRegionCoordination = new ZkOpenRegionCoordination(csm, server.getZooKeeper()) { @Override public boolean transitionToOpened(final HRegion r, OpenRegionDetails ord) throws IOException { ZooKeeperWatcher zkw = server.getZooKeeper(); String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); try { ZKUtil.deleteNodeFailSilent(zkw, node); } catch (KeeperException e) { throw new RuntimeException("Ugh failed delete of " + node, e); } return super.transitionToOpened(r, ord); } }; OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd, openRegionCoordination, zkCrd); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); handler.process(); rss.getRegionsInTransitionInRS().put( hri.getEncodedNameAsBytes(), Boolean.TRUE); ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); handler.process(); } catch (IOException ioe) { } finally { HRegion.closeHRegion(region); } assertTrue("region server should have aborted", server.isAborted()); }
@Override public void process() throws IOException { boolean openSuccessful = false; boolean transitionedToOpening = false; final String regionName = regionInfo.getRegionNameAsString(); HRegion region = null; try { if (this.server.isStopped() || this.rsServices.isStopping()) { return; } final String encodedName = regionInfo.getEncodedName(); if (this.rsServices.getFromOnlineRegions(encodedName) != null) { LOG.error("Region " + encodedName + " was already online when we started processing the opening. " + "Marking this new attempt as failed"); return; } if (!isRegionStillOpening()){ LOG.error("Region " + encodedName + " opening cancelled"); return; } if (useZKForAssignment && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) { LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName); return; } transitionedToOpening = true; region = openRegion(); if (region == null) { return; } boolean failed = true; if (!useZKForAssignment || coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) { if (updateMeta(region)) { failed = false; } } if (failed || this.server.isStopped() || this.rsServices.isStopping()) { return; } if (!isRegionStillOpening() || (useZKForAssignment && !coordination.transitionToOpened(region, ord))) { return; } this.rsServices.addToOnlineRegions(region); openSuccessful = true; LOG.debug("Opened " + regionName + " on " + this.server.getServerName()); } finally { if (!openSuccessful) { doCleanUpOnFailedOpen(region, transitionedToOpening, ord); } final Boolean current = this.rsServices.getRegionsInTransitionInRS(). remove(this.regionInfo.getEncodedNameAsBytes()); if (openSuccessful) { if (current == null) { LOG.error("Bad state: we've just opened a region that was NOT in transition. Region=" + regionName); } else if (Boolean.FALSE.equals(current)) { LOG.error("Race condition: we've finished to open a region, while a close was requested " + " on region=" + regionName + ". It can be a critical error, as a region that" + " should be closed is now opened. Closing it now"); cleanupFailedOpen(region); } } } }
OpenRegionHandler extends EventHandler { @Override public void process() throws IOException { boolean openSuccessful = false; boolean transitionedToOpening = false; final String regionName = regionInfo.getRegionNameAsString(); HRegion region = null; try { if (this.server.isStopped() || this.rsServices.isStopping()) { return; } final String encodedName = regionInfo.getEncodedName(); if (this.rsServices.getFromOnlineRegions(encodedName) != null) { LOG.error("Region " + encodedName + " was already online when we started processing the opening. " + "Marking this new attempt as failed"); return; } if (!isRegionStillOpening()){ LOG.error("Region " + encodedName + " opening cancelled"); return; } if (useZKForAssignment && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) { LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName); return; } transitionedToOpening = true; region = openRegion(); if (region == null) { return; } boolean failed = true; if (!useZKForAssignment || coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) { if (updateMeta(region)) { failed = false; } } if (failed || this.server.isStopped() || this.rsServices.isStopping()) { return; } if (!isRegionStillOpening() || (useZKForAssignment && !coordination.transitionToOpened(region, ord))) { return; } this.rsServices.addToOnlineRegions(region); openSuccessful = true; LOG.debug("Opened " + regionName + " on " + this.server.getServerName()); } finally { if (!openSuccessful) { doCleanUpOnFailedOpen(region, transitionedToOpening, ord); } final Boolean current = this.rsServices.getRegionsInTransitionInRS(). remove(this.regionInfo.getEncodedNameAsBytes()); if (openSuccessful) { if (current == null) { LOG.error("Bad state: we've just opened a region that was NOT in transition. Region=" + regionName); } else if (Boolean.FALSE.equals(current)) { LOG.error("Race condition: we've finished to open a region, while a close was requested " + " on region=" + regionName + ". It can be a critical error, as a region that" + " should be closed is now opened. Closing it now"); cleanupFailedOpen(region); } } } } }
OpenRegionHandler extends EventHandler { @Override public void process() throws IOException { boolean openSuccessful = false; boolean transitionedToOpening = false; final String regionName = regionInfo.getRegionNameAsString(); HRegion region = null; try { if (this.server.isStopped() || this.rsServices.isStopping()) { return; } final String encodedName = regionInfo.getEncodedName(); if (this.rsServices.getFromOnlineRegions(encodedName) != null) { LOG.error("Region " + encodedName + " was already online when we started processing the opening. " + "Marking this new attempt as failed"); return; } if (!isRegionStillOpening()){ LOG.error("Region " + encodedName + " opening cancelled"); return; } if (useZKForAssignment && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) { LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName); return; } transitionedToOpening = true; region = openRegion(); if (region == null) { return; } boolean failed = true; if (!useZKForAssignment || coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) { if (updateMeta(region)) { failed = false; } } if (failed || this.server.isStopped() || this.rsServices.isStopping()) { return; } if (!isRegionStillOpening() || (useZKForAssignment && !coordination.transitionToOpened(region, ord))) { return; } this.rsServices.addToOnlineRegions(region); openSuccessful = true; LOG.debug("Opened " + regionName + " on " + this.server.getServerName()); } finally { if (!openSuccessful) { doCleanUpOnFailedOpen(region, transitionedToOpening, ord); } final Boolean current = this.rsServices.getRegionsInTransitionInRS(). remove(this.regionInfo.getEncodedNameAsBytes()); if (openSuccessful) { if (current == null) { LOG.error("Bad state: we've just opened a region that was NOT in transition. Region=" + regionName); } else if (Boolean.FALSE.equals(current)) { LOG.error("Race condition: we've finished to open a region, while a close was requested " + " on region=" + regionName + ". It can be a critical error, as a region that" + " should be closed is now opened. Closing it now"); cleanupFailedOpen(region); } } } } OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, HRegionInfo regionInfo,
      HTableDescriptor htd, OpenRegionCoordination coordination,
      OpenRegionCoordination.OpenRegionDetails ord); protected  OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, final HRegionInfo regionInfo,
      final HTableDescriptor htd, EventType eventType,
      OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord); }
OpenRegionHandler extends EventHandler { @Override public void process() throws IOException { boolean openSuccessful = false; boolean transitionedToOpening = false; final String regionName = regionInfo.getRegionNameAsString(); HRegion region = null; try { if (this.server.isStopped() || this.rsServices.isStopping()) { return; } final String encodedName = regionInfo.getEncodedName(); if (this.rsServices.getFromOnlineRegions(encodedName) != null) { LOG.error("Region " + encodedName + " was already online when we started processing the opening. " + "Marking this new attempt as failed"); return; } if (!isRegionStillOpening()){ LOG.error("Region " + encodedName + " opening cancelled"); return; } if (useZKForAssignment && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) { LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName); return; } transitionedToOpening = true; region = openRegion(); if (region == null) { return; } boolean failed = true; if (!useZKForAssignment || coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) { if (updateMeta(region)) { failed = false; } } if (failed || this.server.isStopped() || this.rsServices.isStopping()) { return; } if (!isRegionStillOpening() || (useZKForAssignment && !coordination.transitionToOpened(region, ord))) { return; } this.rsServices.addToOnlineRegions(region); openSuccessful = true; LOG.debug("Opened " + regionName + " on " + this.server.getServerName()); } finally { if (!openSuccessful) { doCleanUpOnFailedOpen(region, transitionedToOpening, ord); } final Boolean current = this.rsServices.getRegionsInTransitionInRS(). remove(this.regionInfo.getEncodedNameAsBytes()); if (openSuccessful) { if (current == null) { LOG.error("Bad state: we've just opened a region that was NOT in transition. Region=" + regionName); } else if (Boolean.FALSE.equals(current)) { LOG.error("Race condition: we've finished to open a region, while a close was requested " + " on region=" + regionName + ". It can be a critical error, as a region that" + " should be closed is now opened. Closing it now"); cleanupFailedOpen(region); } } } } OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, HRegionInfo regionInfo,
      HTableDescriptor htd, OpenRegionCoordination coordination,
      OpenRegionCoordination.OpenRegionDetails ord); protected  OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, final HRegionInfo regionInfo,
      final HTableDescriptor htd, EventType eventType,
      OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord); HRegionInfo getRegionInfo(); @Override void process(); }
OpenRegionHandler extends EventHandler { @Override public void process() throws IOException { boolean openSuccessful = false; boolean transitionedToOpening = false; final String regionName = regionInfo.getRegionNameAsString(); HRegion region = null; try { if (this.server.isStopped() || this.rsServices.isStopping()) { return; } final String encodedName = regionInfo.getEncodedName(); if (this.rsServices.getFromOnlineRegions(encodedName) != null) { LOG.error("Region " + encodedName + " was already online when we started processing the opening. " + "Marking this new attempt as failed"); return; } if (!isRegionStillOpening()){ LOG.error("Region " + encodedName + " opening cancelled"); return; } if (useZKForAssignment && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) { LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName); return; } transitionedToOpening = true; region = openRegion(); if (region == null) { return; } boolean failed = true; if (!useZKForAssignment || coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open")) { if (updateMeta(region)) { failed = false; } } if (failed || this.server.isStopped() || this.rsServices.isStopping()) { return; } if (!isRegionStillOpening() || (useZKForAssignment && !coordination.transitionToOpened(region, ord))) { return; } this.rsServices.addToOnlineRegions(region); openSuccessful = true; LOG.debug("Opened " + regionName + " on " + this.server.getServerName()); } finally { if (!openSuccessful) { doCleanUpOnFailedOpen(region, transitionedToOpening, ord); } final Boolean current = this.rsServices.getRegionsInTransitionInRS(). remove(this.regionInfo.getEncodedNameAsBytes()); if (openSuccessful) { if (current == null) { LOG.error("Bad state: we've just opened a region that was NOT in transition. Region=" + regionName); } else if (Boolean.FALSE.equals(current)) { LOG.error("Race condition: we've finished to open a region, while a close was requested " + " on region=" + regionName + ". It can be a critical error, as a region that" + " should be closed is now opened. Closing it now"); cleanupFailedOpen(region); } } } } OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, HRegionInfo regionInfo,
      HTableDescriptor htd, OpenRegionCoordination coordination,
      OpenRegionCoordination.OpenRegionDetails ord); protected  OpenRegionHandler(final Server server,
      final RegionServerServices rsServices, final HRegionInfo regionInfo,
      final HTableDescriptor htd, EventType eventType,
      OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord); HRegionInfo getRegionInfo(); @Override void process();  }