text: from __future__ import absolute_import, division, print_function
  import iotbx.phil
  import iotbx.pdb
  import iotbx.mrcfile
  from cctbx import crystal
  from cctbx import maptbx
  from libtbx.utils import Sorry
  import sys, os
  from cctbx.array_family import flex
  from scitbx.math import matrix
  from copy import deepcopy
  from libtbx.utils import null_out
  import libtbx.callbacks # import dependency
  from libtbx import group_args
  from six.moves import range
  from six.moves import zip
  
  master_phil = iotbx.phil.parse("""
  
    input_files {
  
      seq_file = None
         .type = path
         .short_caption = Sequence file
         .help = Sequence file (unique chains only,  \
                 1-letter code, chains separated by \
                 blank line or greater-than sign.)  \
                 Can have chains that are DNA/RNA/protein and\
                 all can be present in one file. \
                 If not supplied, must supply molecular mass or \
                 solvent content.
  
      map_file = None
        .type = path
        .help = File with CCP4-style map
        .short_caption = Map file
  
      half_map_file = None
        .type = path
        .multiple = True
        .short_caption = Half map
        .help = Half map (two should be supplied) for FSC calculation. Must \
                 have grid identical to map_file
  
      ncs_file = None
        .type = path
        .help = File with symmetry information (typically point-group NCS with \
                 the center specified). Typically in  PDB format. \
                Can also be a .ncs_spec file from phenix. \
                Created automatically if symmetry is specified.
        .short_caption = symmetry file
  
      pdb_file = None
        .type = path
        .help = Optional PDB file matching map_file to be offset
  
      pdb_to_restore = None
        .type = path
        .help = Optional PDB file to restore to position matching original \
                map_file.  Used in combination with info_file=xxx.pkl \
                and restored_pdb=yyyy.pdb
        .short_caption = PDB to restore
  
      info_file = None
        .type = path
        .help = Optional pickle file with information from a previous run.\
                Can be used with pdb_to_restore to restore a PDB file to \
                to position matching original \
                map_file.
        .short_caption = Info file
  
      target_ncs_au_file = None
        .type = path
        .help = Optional PDB file to partially define the ncs asymmetric \
                 unit of the map. The coordinates in this file will be used \
                 to mark part of the ncs au and all points nearby that are \
                 not part of another ncs au will be added.
  
       input_weight_map_pickle_file = None
         .type = path
         .short_caption = Input weight map pickle file
         .help = Weight map pickle file
    }
  
    output_files {
  
      magnification_map_file = magnification_map.ccp4
        .type = path
        .help = Input map file with magnification applied.  Only written if\
                  magnification is applied.
        .short_caption = Magnification map file
  
      magnification_ncs_file = magnification_ncs.ncs_spec
        .type = path
        .help = Input NCS with magnification applied.  Only written if\
                  magnification is applied.
        .short_caption = Magnification NCS file
  
      shifted_map_file = shifted_map.ccp4
        .type = path
        .help = Input map file shifted to new origin.
        .short_caption = Shifted map file
  
      shifted_sharpened_map_file = shifted_sharpened_map.ccp4
        .type = path
        .help = Input map file shifted to new origin and sharpened.
        .short_caption = Shifted sharpened map file
  
      sharpened_map_file = sharpened_map.ccp4
        .type = str
        .short_caption = Sharpened map file
        .help = Output sharpened map file, superimposed on the original map.
        .input_size = 400
  
      shifted_pdb_file = shifted_pdb.pdb
        .type = path
        .help = Input pdb file shifted to new origin.
        .short_caption = Shifted pdb file
  
      shifted_ncs_file = shifted_ncs.ncs_spec
        .type = path
        .help = NCS information shifted to new origin.
        .short_caption = Output NCS info file
  
      shifted_used_ncs_file = shifted_used_ncs.ncs_spec
        .type = path
        .help = NCS information (just the part that is used) shifted \
                 to new origin.
        .short_caption = Output used NCS info file
  
      output_directory = segmented_maps
        .type = path
        .help = Directory where output files are to be written \
                  applied.
        .short_caption = Output directory
  
      box_map_file = box_map_au.ccp4
        .type = path
        .help = Output map file with one NCS asymmetric unit, cut out box
        .short_caption = Box NCS map file
  
      box_mask_file = box_mask_au.ccp4
        .type = path
        .help = Output mask file with one NCS asymmetric unit, cut out box
        .short_caption = Box NCS mask file
  
      box_buffer = 5
        .type = int
        .help = Buffer (grid units) around NCS asymmetric unit in box_mask and map
        .short_caption = Box buffer size
  
      au_output_file_stem = shifted_au
        .type = str
        .help = File stem for output map files with one NCS asymmetric unit
        .short_caption = Output au file stem
  
      write_intermediate_maps = False
        .type = bool
        .help = Write out intermediate maps and masks for visualization
        .short_caption = Write intermediate maps
  
      write_output_maps = True
        .type = bool
        .help = Write out maps
        .short_caption = Write maps
  
      remainder_map_file = remainder_map.ccp4
        .type = path
        .help = output map file with remainder after initial regions identified
        .short_caption = Output remainder map file
  
      output_info_file = segment_and_split_map_info.pkl
        .type = path
        .help = Output pickle file with information about map and masks
        .short_caption = Output pickle file
  
      restored_pdb = None
        .type = path
        .help = Output name of PDB restored to position matching original \
                map_file.  Used in combination with info_file=xxx.pkl \
                and pdb_to_restore=xxxx.pdb
        .short_caption = Restored PDB file
  
      output_weight_map_pickle_file = weight_map_pickle_file.pkl
         .type = path
         .short_caption = Output weight map pickle file
         .help = Output weight map pickle file
    }
  
    crystal_info {
  
       chain_type = *None PROTEIN RNA DNA
         .type = choice
         .short_caption = Chain type
         .help = Chain type. Determined automatically from sequence file if \
                 not given. Mixed chain types are fine (leave blank if so).
  
       sequence = None
         .type = str
         .short_caption = Sequence
         .help = Sequence as string
  
       is_crystal = None
         .type = bool
         .short_caption = Is a crystal
         .help = Defines whether this is a crystal (or cryo-EM).\
                  Default is True if use_sg_symmetry=True and False otherwise.
  
       use_sg_symmetry = False
         .type = bool
         .short_caption = Use space-group symmetry
         .help = If you set use_sg_symmetry=True then the symmetry of the space\
                 group will be used. For example in P1 a point at one end of \
                 the \
                 unit cell is next to a point on the other end.  Normally for \
                 cryo-EM data this should be set to False and for crystal data \
                 it should be set to True. This will normally also set the \
                 value of is_crystal (same value as use_sg_symmetry) and \
                 restrict_map_size (False if use_sg_symmetry=True).
  
       resolution = None
         .type = float
         .short_caption = resolution
         .help = Nominal resolution of the map. This is used later to decide on\
                 resolution cutoffs for Fourier inversion of the map. Note: \
                 the resolution is not cut at this value, it is cut at \
                 resolution*d_min_ratio if at all.
  
       space_group = None
         .type = space_group
         .help = Space group (used for boxed maps)
         .style = hidden
  
       unit_cell = None
         .type = unit_cell
         .help = Unit Cell (used for boxed maps)
         .style = hidden
  
       original_unit_cell = None
         .type = unit_cell
         .help = Original unit cell (of input map). Used internally
         .style = hidden
  
       original_unit_cell_grid = None
         .type = ints
         .help = Original unit cell grid (of input map). Used internally
         .style = hidden
  
       molecular_mass = None
         .type = float
         .help = Molecular mass of molecule in Da. Used as alternative method \
                   of specifying solvent content.
         .short_caption = Molecular mass in Da
  
       solvent_content = None
         .type = float
         .help = Solvent fraction of the cell. Used for ID of \
                 solvent content in boxed maps.
         .short_caption = Solvent content
  
       solvent_content_iterations = 3
         .type = int
         .help = Iterations of solvent fraction estimation. Used for ID of \
                 solvent content in boxed maps.
         .short_caption = Solvent fraction iterations
         .style = hidden
  
       wang_radius = None
         .type = float
         .help = Wang radius for solvent identification. \
             Default is 1.5* resolution
         .short_caption = Wang radius
  
       buffer_radius = None
         .type = float
         .help = Buffer radius for mask smoothing. \
             Default is resolution
         .short_caption = Buffer radius
  
       pseudo_likelihood = None
         .type = bool
         .help = Use pseudo-likelihood method for half-map sharpening. \
                 (In development)
         .short_caption = Pseudo-likelihood
         .style = hidden
  
    }
  
    reconstruction_symmetry {
  
       symmetry = None
         .type = str
         .short_caption = Symmetry type
         .help = Symmetry used in reconstruction. For example D7, C3, C2\
            I (icosahedral),T (tetrahedral), or ANY (try everything and \
            use the highest symmetry found). Not needed if ncs_file is supplied. \
  
       include_helical_symmetry = True
         .type = bool
         .short_caption = Include helical symmetry
         .help = You can include or exclude searches for helical symmetry
  
       symmetry_center = None
         .type = floats
         .short_caption = symmetry center
         .help = Center (in A) for symmetry operators (if symmetry is found \
            automatically). \
            If set to None, first guess is the center of the cell and then \
            if that fails, found automatically as the center of the \
            density in the map.
  
       optimize_center = False
         .type = bool
         .short_caption = Optimize symmetry center
         .help = Optimize position of symmetry center. Also checks for center \
                  at (0,0,0) vs center of map
  
       helical_rot_deg = None
         .type = float
         .short_caption = helical rotation
         .help = helical rotation about z in degrees
  
       helical_trans_z_angstrom = None
         .type = float
         .short_caption = helical translation
         .help = helical translation along z in Angstrom units
  
       max_helical_optimizations = 2
         .type = int
         .short_caption = Max helical optimizations
         .help = Number of optimizations of helical parameters\
                 when finding symmetry
  
       max_helical_ops_to_check = 5
         .type = int
         .short_caption = Max helical ops to check
         .help = Number of helical operations in each direction to check \
                 when finding symmetry
  
       max_helical_rotations_to_check = None
         .type = int
         .short_caption = Max helical rotations
         .help = Number of helical rotations to check \
                 when finding symmetry
  
       two_fold_along_x = None
         .type = bool
         .short_caption = D two-fold along x
         .help = Specifies if D or I two-fold is along x (True) or y (False). \
                 If None, both are tried.
  
       smallest_object = None
         .type = float
         .short_caption = Smallest object to consider
         .help = Dimension of smallest object to consider\
                 when finding symmetry. Default is 5 * resolution
  
       score_basis = ncs_score cc *None
         .type = choice
         .short_caption = Symmetry score basis
         .help = Symmetry score basis. Normally ncs_score (sqrt(n)* cc) is \
                 used except for identification of helical symmetry
  
       scale_weight_fractional_translation= 1.05
         .type = float
         .short_caption = Scale on fractional translation
         .help =  Give slight increase in weighting in helical symmetry \
                 search to translations that are a fraction (1/2, 1/3) of \
                 the d-spacing of the peak of intensity in the fourier \
                 transform of the density.
  
  
       random_points = 100
         .type = int
         .short_caption = Random points
         .help = Number of random points in map to examine in finding symmetry
  
       identify_ncs_id = True
         .type = bool
         .short_caption = Identify NCS ID
         .help = If symmetry is not point-group symmetry, try each possible \
                 operator when evaluating symmetry and choose the one that  \
                 results in the most uniform density at symmetry-related points.
  
       min_ncs_cc = 0.75
         .type = float
         .short_caption = Minimum symmetry CC to keep it
         .help =  Minimum symmetry CC to keep operators when identifying \
                   automatically
  
       n_rescore = 5
         .type = int
         .short_caption = symmetry operators to rescore
         .help = Number of symmetry operators to rescore
  
       op_max = 14
         .type = int
         .short_caption = Max operators to try
         .help = If symmetry is ANY, try up to op_max-fold symmetries
  
  
      tol_r = 0.02
        .type = float
        .help = tolerance in rotations for point group or helical symmetry
        .short_caption = Rotation tolerance
  
      abs_tol_t = 2
        .type = float
        .help = tolerance in translations (A) for point group or helical symmetry
        .short_caption = Translation tolerance absolute
  
      max_helical_operators= None
        .type = int
        .help = Maximum helical operators (if extending existing helical\
               operators)
        .short_caption = Maximum helical operators
  
      rel_tol_t = .05
        .type = float
        .help = tolerance in translations (fractional) for point group or \
               helical symmetry
        .short_caption = Translation tolerance fractional
  
      require_helical_or_point_group_symmetry = False
        .type = bool
        .help = normally helical or point-group symmetry (or none) is expected. \
               However in some cases (helical + rotational symmetry for \
               example) this is not needed and is not the case.
        .short_caption = Require helical or point-group or no symmetry
  
  
       }
  
    map_modification {
  
       magnification = None
         .type = float
         .short_caption = Magnification
         .help = Magnification to apply to input map.  Input map grid will be \
                  scaled by magnification factor before anything else is done.
  
       b_iso = None
         .type = float
         .short_caption = Target b_iso
         .help = Target B-value for map (sharpening will be applied to yield \
            this value of b_iso). If sharpening method is not supplied, \
            default is to use b_iso_to_d_cut sharpening.
  
       b_sharpen = None
         .type = float
         .short_caption = Sharpening
         .help = Sharpen with this b-value. Contrast with b_iso that yield a \
             targeted value of b_iso. B_sharpen greater than zero is sharpening.\
             Less than zero is blurring.
  
       b_blur_hires = 200
         .type = float
         .short_caption = high_resolution blurring
         .help = Blur high_resolution data (higher than d_cut) with \
               this b-value. Contrast with b_sharpen applied to data up to\
               d_cut. \
               Note on defaults: If None and b_sharpen is positive (sharpening) \
               then high-resolution data is left as is (not sharpened). \
               If None and b_sharpen is negative (blurring) high-resolution data\
               is also blurred.
  
       resolution_dependent_b = None
         .type = floats
         .short_caption = resolution_dependent b
         .help = If set, apply resolution_dependent_b (b0 b1 b2). \
               Log10(amplitudes) will start at 1, change to b0 at half \
               of resolution specified, changing linearly, \
               change to b1/2 at resolution specified, \
               and change to b1/2+b2 at d_min_ratio*resolution
  
       normalize_amplitudes_in_resdep = False
         .type = bool
         .short_caption = Normalize amplitudes in resdep
         .help = Normalize amplitudes in resolution-dependent sharpening
  
       d_min_ratio = 0.833
         .type = float
         .short_caption = Sharpen d_min ratio
         .help = Sharpening will be applied using d_min equal to \
               d_min_ratio times resolution. Default is 0.833
  
       scale_max = 100000
         .type = float
         .short_caption = Scale_max
         .help = Scale amplitudes from inverse FFT to yield maximum of this value
  
       input_d_cut = None
         .type = float
         .short_caption = d_cut
         .help = High-resolution limit for sharpening
  
       rmsd = None
         .type = float
         .short_caption = RMSD of model
         .help = RMSD of model to true model (if supplied).  Used to \
               estimate expected fall-of with resolution of correct part \
               of model-based map. If None, assumed to be resolution \
               times rmsd_resolution_factor.
  
       rmsd_resolution_factor = 0.25
         .type = float
         .short_caption = rmsd resolution factor
          .help = default RMSD is resolution times resolution factor
  
       fraction_complete = None
         .type = float
         .short_caption = Completeness model
         .help = Completness of model (if supplied).  Used to \
               estimate correct part \
               of model-based map. If None, estimated from max(FSC).
  
       regions_to_keep = None
         .type = int
         .short_caption = Regions to keep
         .help = You can specify a limit to the number of regions to keep\
                  when generating the asymmetric unit of density.
  
       auto_sharpen = True
         .type = bool
         .short_caption = Automatically determine sharpening
         .help = Automatically determine sharpening using kurtosis maximization\
                   or adjusted surface area
  
       auto_sharpen_methods = no_sharpening b_iso *b_iso_to_d_cut \
                              resolution_dependent model_sharpening \
                              half_map_sharpening target_b_iso_to_d_cut None
  
         .type = choice(multi=True)
         .short_caption = Sharpening methods
         .help = Methods to use in sharpening. b_iso searches for b_iso to \
            maximize sharpening target (kurtosis or adjusted_sa). \
            b_iso_to_d_cut applies b_iso only up to resolution specified, with \
            fall-over of k_sharpen.  Resolution dependent adjusts 3 parameters \
            to sharpen variably over resolution range. Default is \
            b_iso_to_d_cut .  target_b_iso_to_d_cut uses target_b_iso_ratio \
            to set b_iso.
  
       box_in_auto_sharpen = False
         .type = bool
         .short_caption = Use box for auto_sharpening
         .help = Use a representative box of density for initial \
                  auto-sharpening instead of the entire map.
  
       density_select_in_auto_sharpen = True
         .type = bool
         .short_caption = density_select to choose box
         .help = Choose representative box of density for initial \
                  auto-sharpening with density_select method \
                  (choose region where there is high density). \
                 Normally use this as well as density_select=True which \
                 carries out density_select at start of segmentation.
  
       density_select_threshold_in_auto_sharpen = None
         .type = float
         .short_caption = density_select threshold to choose box
         .help = Threshold for density select choice of box. Default is 0.05. \
                 If your map has low overall contrast you might need to make this\
                 bigger such as 0.2.
  
  
       allow_box_if_b_iso_set = False
         .type = bool
         .short_caption = Allow box if b_iso set
         .help = Allow box_in_auto_sharpen (if set to True) even if \
                 b_iso is set. Default is to set box_n_auto_sharpen=False \
                 if b_iso is set.
  
       soft_mask = True
         .type = bool
         .help = Use soft mask (smooth change from inside to outside with radius\
               based on resolution of map). Required if you use half-map \
               sharpening without a model, otherwise optional.
         .short_caption = Soft mask
  
       use_weak_density = False
         .type = bool
         .short_caption = Use box with poor density
         .help = When choosing box of representative density, use poor \
                 density (to get optimized map for weaker density)
  
       discard_if_worse = None
         .type = bool
         .short_caption = Discard sharpening if worse
         .help = Discard sharpening if worse
  
       local_sharpening = None
         .type = bool
         .short_caption = Local sharpening
         .help = Sharpen locally using overlapping regions. \
                 NOTE: Best to turn off local_aniso_in_local_sharpening \
                 if symmetry is present.\
                 If local_aniso_in_local_sharpening is True and symmetry is \
                 present this can distort the map for some symmetry copies \
                 because an anisotropy correction is applied\
                 based on local density in one copy and is transferred without \
                 rotation to other copies.
  
       local_aniso_in_local_sharpening = None
         .type = bool
         .short_caption = Local anisotropy
         .help = Use local anisotropy in local sharpening.  \
                 Default is True unless symmetry is present.
  
       overall_before_local = True
         .type = bool
         .short_caption = Overall before local
         .help = Apply overall scaling before local scaling
  
       select_sharpened_map = None
         .type = int
         .short_caption = Sharpened map to use
         .help = Select a single sharpened map to use
  
       read_sharpened_maps = None
         .type = bool
         .short_caption = Read sharpened maps
         .help = Read in previously-calculated sharpened maps
  
       write_sharpened_maps = None
         .type = bool
         .short_caption = Write sharpened maps
         .help = Write out local sharpened maps
  
       smoothing_radius = None
         .type = float
         .short_caption = Smoothing radius
         .help = Sharpen locally using smoothing_radius. Default is 2/3 of \
                   mean distance between centers for sharpening
  
       box_center = None
         .type = floats
         .short_caption = Center of box
         .help = You can specify the center of the box (A units)
  
       box_size = 40 40 40
         .type = ints
         .short_caption = Size of box
         .help = You can specify the size of the boxes to use (grid units)
  
       target_n_overlap = 10
         .type = int
         .short_caption = Target overlap of boxes
         .help = You can specify the targeted overlap of boxes in local \
             sharpening
  
       restrict_map_size = None
         .type = bool
         .short_caption = Restrict box map size
         .help = Restrict box map to be inside full map (required for cryo-EM data).\
                 Default is True if use_sg_symmetry=False.
  
       restrict_z_turns_for_helical_symmetry = 1
         .type = float
         .short_caption = Restrict Z turns for helical symmetry
         .help = Restrict Z turns for helical symmetry.  Number of \
                 turns of helix going each direction in Z is specified.
  
       restrict_z_distance_for_helical_symmetry = None
         .type = float
         .short_caption = Restrict Z distance for helical symmetry
         .help = Restrict Z distance (+/- this distance from center) \
                for helical symmetry.
  
       remove_aniso = True
         .type = bool
         .short_caption = Remove aniso
         .help = You can remove anisotropy (overall and locally) during sharpening
  
       cc_cut = 0.2
         .type = float
         .short_caption = Min reliable CC in half-maps
         .help = Estimate of minimum highly reliable CC in half-map FSC. Used\
                 to decide at what CC value to smooth the remaining CC values.
  
       max_cc_for_rescale = 0.2
         .type = float
         .short_caption = Max CC for rescaleMin reliable CC in half-maps
         .help = Used along with cc_cut and scale_using_last to correct for \
                 small errors in FSC estimation at high resolution.  If the \
                 value of FSC near the high-resolution limit is above \
                 max_cc_for_rescale, assume these values are correct and do not \
                 correct them.
  
       scale_using_last = 3
         .type = int
         .short_caption = Last N bins in FSC assumed to be about zero
         .help = If set, assume that the last scale_using_last bins in the FSC \
            for half-map or model sharpening are about zero (corrects for  \
            errors in the half-map process).
  
       max_box_fraction = 0.5
         .type = float
         .short_caption = Max size of box for auto_sharpening
         .help = If box is greater than this fraction of entire map, use \
                  entire map.
  
       density_select_max_box_fraction = 0.95
         .type = float
         .short_caption = Max size of box for density_select
         .help = If box is greater than this fraction of entire map, use \
                  entire map for density_select. Default is 0.95
  
       mask_atoms = True
         .type = bool
         .short_caption = Mask atoms
         .help = Mask atoms when using model sharpening
  
       mask_atoms_atom_radius = 3
         .type =float
         .short_caption = Mask radius
         .help = Mask for mask_atoms will have mask_atoms_atom_radius
  
       value_outside_atoms = None
         .type = str
         .short_caption = Value outside atoms
         .help = Value of map outside atoms (set to 'mean' to have mean \
                  value inside and outside mask be equal)
  
       k_sharpen = 10
         .type = float
         .short_caption = sharpening transition
         .help = Steepness of transition between sharpening (up to resolution \
             ) and not sharpening (d < resolution).  Note: for blurring, \
             all data are blurred (regardless of resolution), while for \
             sharpening, only data with d about resolution or lower are \
             sharpened. This prevents making very high-resolution data too \
             strong.  Note 2: if k_sharpen is zero, then no \
             transition is applied and all data is sharpened or blurred. \
             Note 3: only used if b_iso is set.
  
       iterate = False
         .type = bool
         .short_caption = Iterate auto-sharpening
         .help = You can iterate auto-sharpening. This is useful in cases where \
                   you do not specify the solvent content and it is not \
                   accurately estimated until sharpening is optimized.
  
       optimize_b_blur_hires = False
         .type = bool
         .short_caption = Optimize value of b_blur_hires
         .help = Optimize value of b_blur_hires. \
                  Only applies for auto_sharpen_methods b_iso_to_d_cut and \
                  b_iso. This is normally carried out and helps prevent \
                  over-blurring at high resolution if the same map is \
                  sharpened more than once.
  
       optimize_d_cut = None
         .type = bool
         .short_caption = Optimize value of d_cut
         .help = Optimize value of d_cut. \
                  Only applies for auto_sharpen_methods b_iso_to_d_cut and \
                  b_iso. Not normally carried out.
  
       adjust_region_weight = True
         .type = bool
         .short_caption = Adjust region weight
         .help = Adjust region_weight to make overall change in surface area \
                 equal to overall change in normalized regions over the range \
                 of search_b_min to search_b_max using b_iso_to_d_cut.
  
       region_weight_method = initial_ratio *delta_ratio b_iso
         .type = choice
         .short_caption = Region weight method
         .help = Method for choosing region_weights. Initial_ratio uses \
                 ratio of surface area to regions at low B value.  Delta \
                 ratio uses change in this ratio from low to high B. B_iso \
                 uses resolution-dependent b_iso (not weights) with the \
                 formula b_iso=5.9*d_min**2
  
       region_weight_factor = 1.0
         .type = float
         .short_caption = Region weight factor
         .help = Multiplies region_weight after calculation with \
                 region_weight_method above
  
       region_weight_buffer = 0.1
         .type = float
         .short_caption = Region weight factor buffer
         .help = Region_weight adjusted to be region_weight_buffer \
                 away from minimum or maximum values
  
       region_weight_default = 30.
         .type = float
         .short_caption = Region weight default
         .help = Region_weight adjusted to be region_weight_default\
                 if no information available
  
       target_b_iso_ratio = 5.9
         .type = float
         .short_caption = Target b_iso ratio
         .help = Target b_iso ratio : b_iso is estimated as \
                 target_b_iso_ratio * resolution**2
  
       signal_min = 3.0
         .type = float
         .short_caption = Minimum signal
         .help = Minimum signal in estimation of optimal b_iso.  If\
                  not achieved, use any other method chosen.
  
       target_b_iso_model_scale = 0.
         .type = float
         .short_caption = scale on target b_iso ratio for model
         .help = For model sharpening, the target_biso is scaled \
                  (normally zero).
  
       search_b_min = -100
         .type = float
         .short_caption = Low bound for b_iso search
         .help = Low bound for b_iso search.
  
       search_b_max = 300
         .type = float
         .short_caption = High bound for b_iso search
         .help = High bound for b_iso search.
  
       search_b_n = 21
         .type = int
         .short_caption = Number of b_iso values to search
         .help = Number of b_iso values to search.
  
       residual_target = 'adjusted_sa'
         .type = str
         .short_caption = Residual target
         .help = Target for maximization steps in sharpening.  \
            Can be kurtosis or adjusted_sa (adjusted surface area)
  
       sharpening_target = 'adjusted_sa'
         .type = str
         .short_caption = Overall sharpening target
         .help = Overall target for sharpening.  Can be kurtosis or adjusted_sa \
            (adjusted surface area).  Used to decide which sharpening approach \
            is used. Note that during optimization, residual_target is used \
            (they can be the same.)
  
       region_weight = 40
         .type = float
         .short_caption = Region weighting
         .help = Region weighting in adjusted surface area calculation.\
              Score is surface area minus region_weight times number of regions.\
              Default is 40. A smaller value will give more sharpening.
  
       sa_percent = 30.
         .type = float
         .short_caption = Percent of target regions in adjusted_sa
         .help = Percent of target regions used in calulation of adjusted \
           surface area.  Default is 30.
  
       fraction_occupied = 0.20
         .type = float
         .short_caption = Fraction of molecular volume inside contours
         .help = Fraction of molecular volume targeted to be inside contours. \
             Used to set contour level. Default is 0.20
  
        n_bins = 20
          .type = int
          .short_caption = Resolution bins
          .help = Number of resolution bins for sharpening. Default is 20.
  
        max_regions_to_test = 30
          .type = int
          .short_caption = Max regions to test
          .help = Number of regions to test for surface area in adjusted_sa \
                  scoring of sharpening
  
        eps = None
          .type = float
          .short_caption = Shift used in calculation of derivatives for \
             sharpening maximization.  Default is 0.01 for kurtosis and 0.5 for \
             adjusted_sa.
  
        k_sol = 0.35
          .type = float
          .help = k_sol value for model map calculation. IGNORED (Not applied)
          .short_caption = k_sol IGNORED
          .style = hidden
  
        b_sol = 50
          .type = float
          .help = b_sol value for model map calculation. IGNORED (Not applied)
          .short_caption = b_sol IGNORED
          .style = hidden
    }
  
    segmentation {
  
      select_au_box = None
        .type = bool
        .help = Select box containing at least one representative region of \
              the map. Also select just symmetry operators relevant to that box. \
                Default is true if number of operators is at least \
                n_ops_to_use_au_box
        .short_caption = select au box
  
      n_ops_to_use_au_box = 25
        .type = int
        .help = If number of operators is this big or more and \
                select_au_box is None, set it to True.
        .short_caption = N ops to use au_box
  
      n_au_box = 5
        .type = int
        .help = Number of symmetry copies to try and get inside au_box
        .short_caption = N au box
  
  
      lower_bounds = None
        .type = ints
        .help = You can select a part of your map for analysis with \
                lower_bounds and upper_bounds.
        .short_caption = Lower bounds
  
      upper_bounds = None
        .type = ints
        .help = You can select a part of your map for analysis with \
                lower_bounds and upper_bounds.
        .short_caption = Upper bounds
  
  
      density_select = True
        .type = bool
        .help = Run map_box with density_select=True to cut out the region \
                in the input map that contains density. Useful if the input map \
                is much larger than the structure. Done before segmentation is\
                carried out.
        .short_caption = Trim map to density
  
      density_select_threshold = 0.05
        .type = float
        .help = Choose region where density is this fraction of maximum or greater
        .short_caption = threshold for density_select
  
      get_half_height_width = None
        .type = bool
        .help = Use 4 times half-width at half-height as estimate of max size
        .short_caption = Half-height width estimation
  
      box_ncs_au = True
        .type = bool
        .help = Box the map containing just the au of the map
        .short_caption = Box NCS au
  
      cell_cutoff_for_solvent_from_mask = 150
        .type = float
        .help = For cells with average edge over this cutoff, use the\
                 low resolution mask (backup) method for solvent estimation
        .short_caption = Cell cutoff for solvent_from_mask
  
      mask_padding_fraction = 0.025
        .type = float
        .help = Adjust threshold of standard deviation map in low resolution \
              mask identification of solvent content to give this much more \
              inside mask than would be obtained with the value of\
               fraction_of_max_mask_threshold.
        .short_caption = Mask padding fraction
  
      fraction_of_max_mask_threshold = .05
        .type = float
        .help = threshold of standard deviation map in low resolution mask \
               identification of solvent content.
        .short_caption = Fraction of max mask_threshold
  
      mask_threshold = None
        .type = float
        .help = threshold in identification of overall mask. If None, guess \
                 volume of molecule from sequence and symmetry copies.
        .short_caption = Density select threshold
  
      grid_spacing_for_au = 3
        .type = int
        .help = Grid spacing for asymmetric unit when constructing asymmetric unit.
        .short_caption = Grid spacing for constructing asymmetric unit
  
      radius = None
        .type = float
        .help = Radius for constructing asymmetric unit.
        .short_caption = Radius for constructing asymmetric unit
  
  
      value_outside_mask = 0.0
        .type = float
        .help = Value to assign to density outside masks
        .short_caption = Value outside mask
  
      density_threshold = None
        .type = float
        .short_caption = Density threshold
        .help = Threshold density for identifying regions of density. \
               Applied after normalizing the density in the region of \
               the molecule to an rms of 1 and mean of zero.
  
      starting_density_threshold = None
        .type = float
        .short_caption = Starting density threshold
        .help = Optional guess of threshold density
  
      iteration_fraction = 0.2
        .type = float
        .short_caption = Iteration fraction
        .help = On iteration of finding regions, assume target volume is \
                this fraction of the value on previous iteration
  
      max_overlap_fraction = 0.05
        .type = float
        .short_caption = Max overlap
        .help = Maximum fractional overlap allowed to density in another \
                asymmetric unit. Definition of a bad region.
  
      remove_bad_regions_percent = 1
        .type = float
        .short_caption = Remove worst overlapping regions
        .help = Remove the worst regions that are part of more than one NCS \
             asymmetric unit, up to remove_bad_regions_percent of the total
  
      require_complete = True
        .type = bool
        .short_caption = Require all symmetry copies to be represented for a region
        .help =  Require all symmetry copies to be represented for a region
  
      split_if_possible = True
        .type = bool
        .short_caption = Split regions if mixed
        .help = Split regions that are split in some symmetry copies.\
                If None, split if most copies are split.
  
      write_all_regions = False
        .type = bool
        .short_caption = Write all regions
        .help = Write all regions to ccp4 map files.
  
      max_per_au = None
        .type = int
        .short_caption = Max regions in au
        .help = Maximum number of regions to be kept in the NCS asymmetric unit
  
      max_per_au_ratio = 5.
        .type = int
        .short_caption = Max ratio of regions to expected
        .help = Maximum ratio of number of regions to be kept in the \
           NCS asymmetric unit to those expected
  
      min_ratio_of_ncs_copy_to_first = 0.5
        .type = float
        .short_caption = Minimum ratio of ncs copy to first
        .help = Minimum ratio of the last ncs_copy region size to maximum
  
      min_ratio = 0.1
        .type = float
        .short_caption = Minimum ratio to keep
        .help = Minimum ratio of region size to maximum to keep it
  
      max_ratio_to_target = 3
        .type = float
        .help = Maximum ratio of grid points in top region to target
        .short_caption = Max ratio to target
  
      min_ratio_to_target = 0.3
        .type = float
        .help = Minimum ratio of grid points in top region to target
        .short_caption = Min ratio to target
  
      min_volume = 10
        .type = int
        .help = Minimum region size to consider (in grid points)
        .short_caption = Minimum region size
  
      residues_per_region = 50
        .type = float
        .help = Target number of residues per region
        .short_caption = Residues per region
  
      seeds_to_try = 10
        .type = int
        .help = Number of regions to try as centers
        .short_caption = Seeds to try
  
      iterate_with_remainder = True
        .type = bool
        .short_caption = Iterate
        .help = Iterate looking for regions based on remainder from first analysis
  
      weight_rad_gyr = 0.1
        .type = float
        .short_caption = Weight on radius of gyration
        .help = Weight on radius of gyration of group of regions in NCS AU \
                 relative to weight on closeness to neighbors.  Normalized to\
                 largest cell dimension with weight=weight_rad_gyr*300/cell_max
  
      expand_size = None
        .type = int
        .help = Grid points to expand size of regions when excluding for next \
                 round. If None, set to approx number of grid points to get \
                 expand_target below
        .short_caption = Expand size
  
      expand_target = 1.5
        .type = float
        .help = Target expansion of regions (A)
        .short_caption = Expand target
  
      mask_additional_expand_size = 1
        .type = int
        .help = Mask expansion in addition to expand_size for final map
        .short_caption = Mask additional expansion
  
      mask_expand_ratio = 1
        .type = int
        .help = Mask expansion relative to resolution for save_box_map_ncs_au
        .short_caption = Mask expand ratio
  
      exclude_points_in_ncs_copies = True
        .type = bool
        .help = Exclude points that are in symmetry copies when creating NCS au. \
                 Does not apply if add_neighbors=True
        .short_caption = Exclude points in symmetry copies
  
      add_neighbors = True
        .type = bool
        .help = Add neighboring regions around the au. Turns off \
             exclude_points_in_ncs_copies also.
        .short_caption = Add neighbors
  
      add_neighbors_dist = 1.
        .type = float
        .help = Max increase in radius of gyration by adding region to keep it.
        .short_caption = Add neighbors dist
    }
  
     control {
       verbose = False
          .type = bool
          .help = '''Verbose output'''
          .short_caption = Verbose output
  
       shift_only = None
          .type = bool
          .short_caption = Shift only
          .help = Shift map and half_maps and stop
  
       sharpen_only = None
          .type = bool
          .short_caption = Sharpen only
          .help = Sharpen map and stop
  
       check_ncs = None
         .type = bool
         .short_caption = Check NCS
         .help = Check the NCS symmetry by estimating NCS correlation and stop
  
       resolve_size = None
          .type = int
          .help = "Size of resolve to use. "
          .style = hidden
  
        quick = True
          .type = bool
          .help = Run quickly if possible
          .short_caption = Quick run
  
       memory_check = True
          .type = bool
          .help = Map-to-model checks to make sure you have enough memory on \
                    your machine to run.  You can disable this by setting this \
                    keyword to False. The estimates are approximate so it is \
                    possible your job could run even if the check fails.  Note \
                    the check does not take any other uses of the memory on \
                    your machine into account.
          .short_caption = Memory check
  
       save_box_map_ncs_au = False
         .type = bool
         .help = Controls whether the map_box ncs_au is saved. Internal use only
         .style = hidden
  
       write_files = True
         .type = bool
         .help = Controls whether files are written
         .short_caption = Write files
  
        multiprocessing = *multiprocessing sge lsf pbs condor pbspro slurm
          .type = choice
          .short_caption = multiprocessing type
          .help = Choices are multiprocessing (single machine) or queuing systems
  
        queue_run_command = None
          .type = str
          .short_caption = Queue run command
          .help = run command for queue jobs. For example qsub.
  
        nproc = 1
          .type = int
          .short_caption = Number of processors
          .help = Number of processors to use
          .style = renderer:draw_nproc_widget bold
  
  
     }
  """, process_includes=True)
  master_params = master_phil
  
  class map_and_b_object:
    def __init__(self,
         map_data=None,
        starting_b_iso=None,
        final_b_iso=None):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
  
  
  class pdb_info_object:
    def __init__(self,
      file_name=None,
      n_residues=None,
      ):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      import time
      self.init_asctime=time.asctime()
  
    def show_summary(self,out=sys.stdout):
      print("PDB file:%s" %(self.file_name), end=' ', file=out)
      if self.n_residues:
        print("   Residues: %d" %(self.n_residues), file=out)
      else:
        print(file=out)
  
  class seq_info_object:
    def __init__(self,
      file_name=None,
      sequence=None,
      n_residues=None,
      ):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      import time
      self.init_asctime=time.asctime()
  
    def show_summary(self,out=sys.stdout):
      if self.file_name:
        print("Sequence file:%s" %(self.file_name), end=' ', file=out)
      if self.n_residues:
        print("   Residues: %d" %(self.n_residues), file=out)
      else:
        print(file=out)
  
  
  class ncs_info_object:
    def __init__(self,
      file_name=None,
      number_of_operators=None,
      is_helical_symmetry=None,
      original_number_of_operators=None,
      ):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      import time
      self.init_asctime=time.asctime()
      if original_number_of_operators is None:
         self.original_number_of_operators=number_of_operators
  
      self._has_updated_operators=False
  
    def show_summary(self,out=sys.stdout):
      print("NCS file:%s   Operators: %d" %(self.file_name,
        self.number_of_operators), file=out)
      if self.is_helical_symmetry:
        print("Helical symmetry is present", file=out)
  
    def has_updated_operators(self):
      return self._has_updated_operators
  
    def update_number_of_operators(self,number_of_operators=None):
      self.number_of_operators=number_of_operators
      self._has_updated_operators=True
  
    def update_is_helical_symmetry(self,is_helical_symmetry=None):
      self.is_helical_symmetry=is_helical_symmetry
      self._has_updated_operators=True
  
  
  class map_info_object:
    def __init__(self,
      file_name=None,
      origin=None,
      all=None,
      crystal_symmetry=None,
      is_map=None,
      map_id=None,
      b_sharpen=None,
      id=None,
      ):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      import time
      self.init_asctime=time.asctime()
  
    def show_summary(self,out=sys.stdout):
      if self.is_map:
        print("Map file:%s" %(self.file_name), end=' ', file=out)
      else:
        print("Mask file:%s" %(self.file_name), end=' ', file=out)
      if self.id is not None:
        print("ID: %d" %(self.id), end=' ', file=out)
      if self.b_sharpen is not None:
        print("B-sharpen: %7.2f" %(self.b_sharpen), end=' ', file=out)
      if self.map_id is not None:
        print("Map ID: %s" %(self.map_id), file=out)
      else:
        print(file=out)
      if self.origin and self.all:
        print("   Origin: %d  %d  %d   Extent: %d  %d  %d" %(
         tuple(self.origin)+tuple(self.all)), file=out)
      if self.crystal_symmetry:
        print("   Map unit cell: %.1f  %.1f  %.1f    %.1f  %.1f  %.1f " %(
          self.crystal_symmetry.unit_cell().parameters()), file=out)
  
    def lower_upper_bounds(self):
      lower_bounds=self.origin
      upper_bounds=[]
      for a,b in zip(self.origin,self.all):
        upper_bounds.append(a+b)
      return list(self.origin),list(upper_bounds)
  
  class info_object:
    def __init__(self,
        acc=None,
        ncs_obj=None,
        min_b=None,
        max_b=None,
        b_sharpen=None,  # b_sharpen applied to map
        ncs_group_list=None,
        origin_shift=None,
        crystal_symmetry=None, # after density_select
        original_crystal_symmetry=None, # before density_select
        full_crystal_symmetry=None, # from real_map object
        full_unit_cell_grid=None,  # from real_map object
        edited_volume_list=None,
        region_range_dict=None,
        selected_regions=None,
        ncs_related_regions=None,
        self_and_ncs_related_regions=None,
        map_files_written=None,
        bad_region_list=None,
        region_centroid_dict=None,
        original_id_from_id=None,
        remainder_id_dict=None,  # dict relating regions in a remainder object to
        params=None, # input params
        input_pdb_info=None,
        input_map_info=None,
        input_ncs_info=None,
        input_seq_info=None,
        shifted_pdb_info=None,
        shifted_map_info=None,
        shifted_ncs_info=None,
        shifted_used_ncs_info=None,
        n_residues=None,
        solvent_fraction=None,
        output_ncs_au_map_info=None,
        output_ncs_au_mask_info=None,
        output_ncs_au_pdb_info=None,
        output_box_map_info=None,
        output_box_mask_info=None,
        output_region_map_info_list=None,
        output_region_pdb_info_list=None,
        sharpening_info_obj=None,
        box_map_bounds_first=None,
        box_map_bounds_last=None,
        final_output_sharpened_map_file=None,
        box_map_ncs_au=None,
        box_map_ncs_au_crystal_symmetry=None,
      ):
      if not selected_regions: selected_regions=[]
      if not ncs_related_regions: ncs_related_regions=[]
      if not self_and_ncs_related_regions: self_and_ncs_related_regions=[]
      if not map_files_written: map_files_written=[]
      if not output_region_map_info_list: output_region_map_info_list=[]
      if not output_region_pdb_info_list: output_region_pdb_info_list=[]
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
  
      self.object_type="segmentation_info"
      import time
      self.init_asctime=time.asctime()
  
    def set_box_map_ncs_au_map_data(self,box_map_ncs_au_map_data=None,
         box_map_ncs_au_half_map_data_list=None,
         box_map_ncs_au_crystal_symmetry=None):
      self.box_map_ncs_au_map_data=box_map_ncs_au_map_data.deep_copy()
      self.box_map_ncs_au_half_map_data_list=[]
      for hm in box_map_ncs_au_half_map_data_list:
         self.box_map_ncs_au_half_map_data_list.append(hm.deep_copy())
      self.box_map_ncs_au_crystal_symmetry=box_map_ncs_au_crystal_symmetry
      if self.origin_shift and self.origin_shift != (0,0,0):
        self.box_map_ncs_au_map_data=self.shift_map_back(
          map_data=self.box_map_ncs_au_map_data,
          crystal_symmetry=self.box_map_ncs_au_crystal_symmetry,
          shift_cart=self.origin_shift)
  
        new_hm_list=[]
        for hm in self.box_map_ncs_au_half_map_data_list:
          hm=self.shift_map_back(
            map_data=hm,
            crystal_symmetry=self.box_map_ncs_au_crystal_symmetry,
            shift_cart=self.origin_shift)
          new_hm_list.append(hm)
        self.box_map_ncs_au_half_map_data_list=new_hm_list
  
    def shift_map_back(self,map_data=None,
        crystal_symmetry=None,shift_cart=None):
      from scitbx.matrix import col
      new_origin=self.origin_shift_grid_units(crystal_symmetry=crystal_symmetry,
        map_data=map_data,shift_cart=shift_cart,reverse=True)
      new_all=list(col(map_data.all())+col(new_origin))
      shifted_map_data = map_data.deep_copy()
      shifted_map_data.resize(flex.grid(new_origin,new_all))
      return shifted_map_data
  
    def origin_shift_grid_units(self,crystal_symmetry=None,map_data=None,
         shift_cart=None,reverse=False):
      # Get origin shift in grid units from shift_cart
      from scitbx.matrix import col
      cell=crystal_symmetry.unit_cell().parameters()[:3]
      origin_shift_grid=[]
      for s,c,a in zip(shift_cart,cell,map_data.all()):
        if s<0:
          delta=-0.5
        else:
          delta=0.5
        origin_shift_grid.append( int(delta+ a*s/c))
      if reverse:
        return list(-col(origin_shift_grid))
      else:
        return origin_shift_grid
  
  
    def is_segmentation_info_object(self):
      return True
  
    def set_params(self,params):
      self.params=deepcopy(params)
  
    def set_input_seq_info(self,file_name=None,sequence=None,n_residues=None):
      self.input_seq_info=seq_info_object(file_name=file_name,
         sequence=sequence,
         n_residues=n_residues)
  
    def set_input_pdb_info(self,file_name=None,n_residues=None):
      self.input_pdb_info=pdb_info_object(file_name=file_name,
       n_residues=n_residues)
  
    def set_input_ncs_info(self,file_name=None,number_of_operators=None):
      self.input_ncs_info=ncs_info_object(file_name=file_name,
        number_of_operators=number_of_operators)
  
    def update_ncs_info(self,number_of_operators=None,is_helical_symmetry=None,
        shifted=False):
      if shifted:
        ncs_info=self.shifted_ncs_info
      else:
        ncs_info=self.input_ncs_info
      assert ncs_info
      if number_of_operators is not None:
        ncs_info.update_number_of_operators(
          number_of_operators=number_of_operators)
      if is_helical_symmetry is not None:
        ncs_info.update_is_helical_symmetry(
          is_helical_symmetry=is_helical_symmetry)
  
    def set_sharpening_info(self,sharpening_info_obj=None):
       self.sharpening_info_obj=sharpening_info_obj
  
    def set_input_map_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None):
      self.input_map_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        is_map=True)
  
    def set_ncs_obj(self,ncs_obj=None):
      self.ncs_obj=ncs_obj
  
    def set_origin_shift(self,origin_shift=None):
      if not origin_shift: origin_shift=(0,0,0)
      self.origin_shift=tuple(origin_shift)
  
    def set_crystal_symmetry(self,crystal_symmetry):
      self.crystal_symmetry=deepcopy(crystal_symmetry)
  
    def set_original_crystal_symmetry(self,crystal_symmetry):
      self.original_crystal_symmetry=deepcopy(crystal_symmetry)
  
    def set_full_crystal_symmetry(self,crystal_symmetry):
      self.full_crystal_symmetry=deepcopy(crystal_symmetry)
  
    def set_full_unit_cell_grid(self,unit_cell_grid):
      self.full_unit_cell_grid=deepcopy(unit_cell_grid)
  
    def set_box_map_bounds_first_last(self,box_map_bounds_first,
        box_map_bounds_last):
      self.box_map_bounds_first=box_map_bounds_first
      self.box_map_bounds_last=[]
      for l in box_map_bounds_last:
        self.box_map_bounds_last.append(l+1)  # it is one bigger...
  
    def set_accessor(self,acc):
      self.acc=acc
  
    def set_shifted_map_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None,b_sharpen=None):
      self.shifted_map_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        b_sharpen=b_sharpen,
        is_map=True)
  
    def set_shifted_pdb_info(self,file_name=None,n_residues=None):
      self.shifted_pdb_info=pdb_info_object(file_name=file_name,
       n_residues=n_residues)
  
    def set_shifted_ncs_info(self,file_name=None,number_of_operators=None,
         is_helical_symmetry=None):
      self.shifted_ncs_info=ncs_info_object(file_name=file_name,
        number_of_operators=number_of_operators,
        is_helical_symmetry=is_helical_symmetry)
  
    def set_shifted_used_ncs_info(self,file_name=None,number_of_operators=None,
         is_helical_symmetry=None):
      self.shifted_used_ncs_info=ncs_info_object(file_name=file_name,
        number_of_operators=number_of_operators,
        is_helical_symmetry=is_helical_symmetry)
  
    def set_solvent_fraction(self,solvent_fraction):
      self.solvent_fraction=solvent_fraction
  
    def set_n_residues(self,n_residues): # may not be the same as seq file
      self.n_residues=n_residues
  
    def set_output_ncs_au_map_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None):
      self.output_ncs_au_map_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        is_map=True)
  
    def set_output_ncs_au_mask_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None):
      self.output_ncs_au_mask_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        is_map=False)
  
    def set_output_ncs_au_pdb_info(self,file_name=None,n_residues=None):
      self.output_ncs_au_pdb_info=pdb_info_object(file_name=file_name,
       n_residues=n_residues)
  
    def set_output_box_map_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None):
      self.output_box_map_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        is_map=True)
  
    def set_output_box_mask_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None):
      self.output_box_mask_info=map_info_object(file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        is_map=False)
  
    def add_output_region_map_info(self,file_name=None,crystal_symmetry=None,
      origin=None,all=None,map_id=None):
      self.output_region_map_info_list.append(map_info_object(
        file_name=file_name,
        crystal_symmetry=crystal_symmetry,
        origin=origin,
        all=all,
        id=len(self.output_region_map_info_list)+1,
        map_id=map_id,
        is_map=True)
       )
  
    def add_output_region_pdb_info(self,file_name=None,n_residues=None):
      self.output_region_pdb_info_list.append(pdb_info_object(
        file_name=file_name,
        n_residues=n_residues)
       )
  
  
    def show_summary(self,out=sys.stdout):
      print("\n==========  Summary of %s: ========\n" %(self.object_type), file=out)
      print("Created: %s" %(self.init_asctime), file=out)
      print("\nInput files used:\n", file=out)
      if self.input_map_info:
        self.input_map_info.show_summary(out=out)
      if self.input_pdb_info:
        self.input_pdb_info.show_summary(out=out)
      if self.input_ncs_info:
        self.input_ncs_info.show_summary(out=out)
      if self.input_seq_info:
        self.input_seq_info.show_summary(out=out)
  
      print(file=out)
  
      if self.crystal_symmetry:
        print("Working unit cell: %.1f  %.1f  %.1f    %.1f  %.1f  %.1f " %(
          self.crystal_symmetry.unit_cell().parameters()), file=out)
  
      if self.n_residues:
        print("Estimated total number of residues: %d" %(self.n_residues), file=out)
  
      if self.solvent_fraction:
        print("Estimated solvent fraction: %5.3f" %(self.solvent_fraction), file=out)
  
      if self.origin_shift and self.origin_shift != (0,0,0):
        print("\nOrigin offset applied: %.1f  %.1f  %.1f" %(self.origin_shift), file=out)
      else:
        print("\nNo origin offset applied", file=out)
  
      if self.shifted_map_info:
        print("\nShifted/sharpened map, pdb and ncs files created "+\
           "(after origin offset):\n", file=out)
        if self.shifted_map_info:
          self.shifted_map_info.show_summary(out=out)
        if self.shifted_pdb_info:
          self.shifted_pdb_info.show_summary(out=out)
        if self.shifted_ncs_info:
          self.shifted_ncs_info.show_summary(out=out)
  
      if self.output_ncs_au_pdb_info:
        print("\nOutput PDB file with dummy atoms representing the NCS AU:", file=out)
        self.output_ncs_au_pdb_info.show_summary(out=out)
  
      if self.output_ncs_au_mask_info or self.output_ncs_au_map_info:
        print("\nOutput map files showing just the NCS AU (same size", end=' ', file=out)
        if self.origin_shift and self.origin_shift != (0,0,0):
          print("\nand location as shifted map files:\n", file=out)
        else:
          print("\nand location as input map:\n", file=out)
  
        if self.output_ncs_au_mask_info:
          self.output_ncs_au_mask_info.show_summary(out=out)
        if self.output_ncs_au_map_info:
          self.output_ncs_au_map_info.show_summary(out=out)
  
      if self.output_box_mask_info or self.output_box_map_info:
        print("\nOutput cut-out map files trimmed to contain just "+\
          "the \nNCS AU (superimposed on", end=' ', file=out)
        if self.origin_shift and self.origin_shift != (0,0,0):
          print("shifted map files, note origin offset):\n", file=out)
        else:
          print("input map, note origin offset):\n", file=out)
  
        if self.output_box_mask_info:
          self.output_box_mask_info.show_summary(out=out)
        if self.output_box_map_info:
          self.output_box_map_info.show_summary(out=out)
  
      if self.output_region_pdb_info_list:
        print("\nOutput PDB files representing one region of connected"+\
          " density.\nThese are useful for marking where to look in cut-out map"+\
          " files.", file=out)
        for output_region_pdb_info in self.output_region_pdb_info_list:
          output_region_pdb_info.show_summary(out=out)
  
      if self.output_region_map_info_list:
        print("\nOutput cut-out map files trimmed to contain just "+\
          "one region of \nconnected density (superimposed on", end=' ', file=out)
        if self.origin_shift and self.origin_shift != (0,0,0):
          print("shifted map files, note origin offset):\n", file=out)
        else:
          print(" input map, note origin offset):\n", file=out)
        for output_region_map_info in self.output_region_map_info_list:
          output_region_map_info.show_summary(out=out)
  
      print("\n"+50*"="+"\n", file=out)
  
  class make_ccp4_map: # just a holder so map_to_structure_factors will run
    def __init__(self,map=None,unit_cell=None):
      self.data=map
      self.unit_cell_parameters=unit_cell.parameters()
      self.space_group_number=1
      self.unit_cell_grid=map.all()
  
    def crystal_symmetry(self):
      return crystal.symmetry(self.unit_cell_parameters,
          self.space_group_number)
  
  
  class b_vs_region_info:
    def __init__(self):
      self.b_iso=0.
      self.b_vs_region_dict={}
      self.sa_sum_v_vs_region_dict={}
      self.sa_nn_vs_region_dict={}
      self.sa_ratio_b_vs_region_dict={}
  
  class box_sharpening_info:
    def __init__(self,tracking_data=None,
        crystal_symmetry=None,
        solvent_fraction=None,
        b_iso=None,
        resolution=None,
        d_min_ratio=None,
        scale_max=None,
        lower_bounds=None,
        upper_bounds=None,
        wrapping=None,
        n_real=None,
        n_buffer=None,
        map_data=None,
        smoothing_radius=None,
        smoothed_box_mask_data=None,
        original_box_map_data=None,
         ):
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      del self.tracking_data # do not save it
      if tracking_data:
        self.crystal_symmetry=tracking_data.crystal_symmetry
        self.solvent_fraction=tracking_data.solvent_fraction
        self.wrapping=tracking_data.params.crystal_info.use_sg_symmetry
  
    def get_gaussian_weighting(self,out=sys.stdout):
      # return a gaussian function centered on center of the map, fall-off
      #   based on smoothing_radius
  
      # Calculate weight map, max near location of centers_ncs_cart
      # U=rmsd**2
      # (b_eff=8*3.14159**2*U)
      #  rmsd is at least distance between centers, not too much bigger than
      #  unit cell size, typically 10-20 A,
      print("\nFall-off of local weight is 1/%6.1f A\n" %(
         self.smoothing_radius), file=out)
      u=self.smoothing_radius**2
  
      from cctbx import xray
      xrs,scatterers=set_up_xrs(crystal_symmetry=self.crystal_symmetry)
      unit_cell=self.crystal_symmetry.unit_cell()
      for xyz_fract in [(0.5,0.5,0.5,)]:
        scatterers.append( xray.scatterer(scattering_type="H", label="H",
          site=xyz_fract, u=u, occupancy=1.0))
      xrs = xray.structure(xrs, scatterers=scatterers)
      f_array,phases=get_f_phases_from_map(map_data=self.map_data,
         crystal_symmetry=self.crystal_symmetry,
         d_min=self.resolution,
         scale_max=self.scale_max,
         d_min_ratio=self.d_min_ratio,
         get_remove_aniso_object=False,# don't need it
         out=out)
  
      weight_f_array=f_array.structure_factors_from_scatterers(
        algorithm = 'direct',
        xray_structure = xrs).f_calc()
  
      weight_map=get_map_from_map_coeffs(map_coeffs=weight_f_array,
        crystal_symmetry=self.crystal_symmetry,n_real=self.map_data.all())
      min_value=weight_map.as_1d().min_max_mean().min
      weight_map=weight_map-min_value # all positive or zero
      max_value=weight_map.as_1d().min_max_mean().max
      weight_map=weight_map/max(1.e-10,max_value)  # normalize; max=1 now
      min_value=1.e-10  # just a small value for all distances far from center
      s = (weight_map <min_value )  # make extra sure every point is above this
      weight_map=weight_map.set_selected(s,min_value)
      return weight_map
  
  
    def remove_buffer_from_bounds(self,minimum=1):
      # back off by n_buffer in each direction, leave at
      # least minimum grid on either side of center
  
      adjusted_lower_bounds,adjusted_upper_bounds=[],[]
      delta_lower_bounds,delta_upper_bounds=[],[]
      for lb,ub in zip(self.lower_bounds,self.upper_bounds):
        sum=lb+ub
        if sum >=0:
          mid=(1+sum)//2
        else:
          mid=(-1+sum)//2
        alb=min(mid-minimum,lb+self.n_buffer)
        aub=max(mid+minimum,ub-self.n_buffer)
        adjusted_lower_bounds.append(alb)
        adjusted_upper_bounds.append(aub)
        delta_lower_bounds.append(alb-lb)
        delta_upper_bounds.append(aub-ub)
      return adjusted_lower_bounds,adjusted_upper_bounds,\
        delta_lower_bounds,delta_upper_bounds
  
  
    def merge_into_overall_map(self,overall_map=None):
      # Smoothly fill out edges of the small map with overall_map
  
      assert self.smoothed_box_mask_data is not None
      assert self.original_box_map_data is not None
  
      self.map_data= (self.map_data * self.smoothed_box_mask_data) + \
         (self.original_box_map_data * (1-self.smoothed_box_mask_data))
  
    def remove_buffer(self,out=sys.stdout):
      # remove the buffer from this box
  
      new_lower_bounds,new_upper_bounds,delta_lower,delta_upper=\
         self.remove_buffer_from_bounds()
  
      cut_out_lower_bounds=[]
      cut_out_upper_bounds=[]
      for o,a,dlb,dub in zip(self.map_data.origin(),self.map_data.all(),
        delta_lower,delta_upper):
       cut_out_lower_bounds.append(o+dlb)
       cut_out_upper_bounds.append(a+dub-1)
      self.map_data,self.crystal_symmetry,\
         self.smoothed_box_mask_data,self.original_box_map_data=cut_out_map(
            map_data=self.map_data,
            crystal_symmetry=self.crystal_symmetry,
            soft_mask=False,
            resolution=self.resolution,
            shift_origin=True,
            min_point=cut_out_lower_bounds,
            max_point=cut_out_upper_bounds,out=out)
      self.lower_bounds=new_lower_bounds
      self.upper_bounds=new_upper_bounds
  
  class sharpening_info:
    def __init__(self,
        tracking_data=None,
        crystal_symmetry=None,
        is_crystal=None,
        sharpening_method=None,
        solvent_fraction=None,
        n_residues=None,
        ncs_copies=None,
        ncs_file=None,
        seq_file=None,
        sequence=None,
        n_real=None,
        region_weight=None,
        n_bins=None,
        eps=None,
        d_min=None,
        d_min_ratio=None,
        scale_max=None,
        input_d_cut=None,
        b_blur_hires=None,
        rmsd=None,
        rmsd_resolution_factor=None,
        k_sol=None,
        b_sol=None,
        fraction_complete=None,
        wrapping=None,
        sharpening_target=None,
        residual_target=None,
        fraction_occupied=None,
        nproc=None,
        multiprocessing=None,
        queue_run_command=None,
        resolution=None, # changed from d_cut
        resolution_dependent_b=None,  # linear sharpening
        normalize_amplitudes_in_resdep=None,  # linear sharpening
        b_sharpen=None,
        b_iso=None,  # expected B_iso after applying b_sharpen
        k_sharpen=None,
        optimize_b_blur_hires=None,
        iterate=None,
        optimize_d_cut=None,
        kurtosis=None,
        adjusted_sa=None,
        sa_ratio=None,
        normalized_regions=None,
        score=None,
        input_weight_map_pickle_file=None,
        output_weight_map_pickle_file=None,
        read_sharpened_maps=None,
        write_sharpened_maps=None,
        select_sharpened_map=None,
        output_directory=None,
        smoothing_radius=None,
        local_sharpening=None,
        local_aniso_in_local_sharpening=None,
        overall_before_local=None,
        use_local_aniso=None,
        original_aniso_obj=None,
        auto_sharpen=None,
        box_in_auto_sharpen=None,
        density_select_in_auto_sharpen=None,
        density_select_threshold_in_auto_sharpen=None,
        use_weak_density=None,
        discard_if_worse=None,
        max_box_fraction=None,
        cc_cut=None,
        max_cc_for_rescale=None,
        scale_using_last=None,
        density_select_max_box_fraction=None,
        mask_atoms=None,
        mask_atoms_atom_radius=None,
        value_outside_atoms=None,
        soft_mask=None,
        allow_box_if_b_iso_set=None,
        search_b_min=None,
        search_b_max=None,
        search_b_n=None,
        adjust_region_weight=None,
        region_weight_method=None,
        region_weight_factor=None,
        region_weight_buffer=None,
        region_weight_default=None,
        target_b_iso_ratio=None,
        signal_min=None,
        target_b_iso_model_scale=None,
        box_sharpening_info_obj=None,
        chain_type=None,
        target_scale_factors=None,
        remove_aniso=None,
        d_min_list=None,
        verbose=None,
        resolve_size=None,
        pdb_inp=None,  # XXX probably do not need this
        local_solvent_fraction=None,
        wang_radius=None,
        buffer_radius=None,
        pseudo_likelihood=None,
        preliminary_sharpening_done=False,
  
          ):
  
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
      del self.tracking_data  # don't need it as part of the object
      del self.box_sharpening_info_obj# don't need it as part of the object
      del self.pdb_inp # don't need it as part of the object
  
      if tracking_data:  # use tracking data information
        self.update_with_tracking_data(tracking_data=tracking_data)
  
      if box_sharpening_info_obj: # update information
        self.update_with_box_sharpening_info(
           box_sharpening_info_obj=box_sharpening_info_obj)
  
      if self.resolution_dependent_b is None:
        self.resolution_dependent_b=[0,0,0]
  
      if self.target_scale_factors and \
          self.sharpening_method!='model_sharpening' \
          and self.sharpening_method!='half_map_sharpening':
        assert self.sharpening_method is None # XXX may want to print out error
        self.sharpening_method='model_sharpening'
  
      if self.sharpening_method=='b_iso' and self.k_sharpen is not None:
        self.k_sharpen=None
  
      if pdb_inp:
          self.sharpening_method='model_sharpening'
          self.box_in_auto_sharpen=True
          self.density_select_in_auto_sharpen=False
          self.sharpening_target='model'
  
    def get_d_cut(self):
      if self.input_d_cut is not None:
         return self.input_d_cut
      else:
         return self.resolution
  
    def get_target_b_iso(self):
      if self.target_b_iso_ratio is None:
        return None
      if self.resolution is None:
        return None
      return self.target_b_iso_ratio*self.resolution**2
  
    def set_resolution_dependent_b(self,
      resolution_dependent_b=None,
      sharpening_method='resolution_dependent'):
      if resolution_dependent_b:
        self.resolution_dependent_b=resolution_dependent_b
      if sharpening_method:
        self.sharpening_method=sharpening_method
  
    def sharpening_is_defined(self):
      if self.sharpening_method is None:
        return False
  
      if self.target_scale_factors:
        return True
  
      if self.sharpening_method=='target_b_iso_to_d_cut':
        return True
  
      if self.b_iso is not None or \
         self.b_sharpen is not None or \
         (self.resolution_dependent_b is not None and
          self.resolution_dependent_b!=[0,0,0]):
        return True
  
      return False
  
    def update_with_box_sharpening_info(self,box_sharpening_info_obj=None):
        if not box_sharpening_info_obj:
          return self
        self.crystal_symmetry=box_sharpening_info_obj.crystal_symmetry
        self.solvent_fraction=box_sharpening_info_obj.solvent_fraction
        self.wrapping=box_sharpening_info_obj.wrapping
        self.n_real=box_sharpening_info_obj.n_real
        return self
  
    def update_with_tracking_data(self,tracking_data=None):
        self.update_with_params(params=tracking_data.params,
           crystal_symmetry=tracking_data.crystal_symmetry,
           solvent_fraction=tracking_data.solvent_fraction,
           n_residues=tracking_data.n_residues,
           ncs_copies=tracking_data.input_ncs_info.number_of_operators)
        return self
  
    def update_with_params(self,params=None,
       crystal_symmetry=None,
       is_crystal=None,
       solvent_fraction=None,
       auto_sharpen=None,
       sharpening_method=None,
       pdb_inp=None,
       half_map_data_list=None,
       n_residues=None,ncs_copies=None):
        self.crystal_symmetry=crystal_symmetry
        self.is_crystal=is_crystal
        self.solvent_fraction=solvent_fraction
        self.auto_sharpen=auto_sharpen
        self.n_residues=n_residues
        self.ncs_copies=ncs_copies
        self.seq_file=params.input_files.seq_file
        self.chain_type=params.crystal_info.chain_type
        self.verbose=params.control.verbose
        self.resolve_size=params.control.resolve_size
        self.multiprocessing=params.control.multiprocessing
        self.nproc=params.control.nproc
        self.queue_run_command=params.control.queue_run_command
  
        self.wrapping=params.crystal_info.use_sg_symmetry
        self.fraction_occupied=params.map_modification.fraction_occupied
        self.sa_percent=params.map_modification.sa_percent
        self.region_weight=params.map_modification.region_weight
        self.max_regions_to_test=params.map_modification.max_regions_to_test
        self.regions_to_keep=params.map_modification.regions_to_keep
        self.d_min_ratio=params.map_modification.d_min_ratio
        self.scale_max=params.map_modification.scale_max
  
        self.input_d_cut=params.map_modification.input_d_cut
        self.b_blur_hires=params.map_modification.b_blur_hires
        self.rmsd=params.map_modification.rmsd
        self.rmsd_resolution_factor=params.map_modification.rmsd_resolution_factor
        self.k_sol=params.map_modification.k_sol
        self.b_sol=params.map_modification.b_sol
        self.fraction_complete=params.map_modification.fraction_complete
        self.resolution=params.crystal_info.resolution  # changed from d_cut
        #  NOTE:
        #  resolution=X-ray resolution or nominal resolution of cryoEM map
        #  high-res cutoff of reflections is d_min*d_min_ratio
        self.buffer_radius=params.crystal_info.buffer_radius
        self.wang_radius=params.crystal_info.wang_radius
        self.pseudo_likelihood=params.crystal_info.pseudo_likelihood
  
        self.max_box_fraction=params.map_modification.max_box_fraction
        self.cc_cut=params.map_modification.cc_cut
        self.max_cc_for_rescale=params.map_modification.max_cc_for_rescale
        self.scale_using_last=params.map_modification.scale_using_last
        self.density_select_max_box_fraction=params.map_modification.density_select_max_box_fraction
        self.mask_atoms=params.map_modification.mask_atoms
        self.mask_atoms_atom_radius=params.map_modification.mask_atoms_atom_radius
        self.value_outside_atoms=params.map_modification.value_outside_atoms
        self.soft_mask=params.map_modification.soft_mask
        self.allow_box_if_b_iso_set=params.map_modification.allow_box_if_b_iso_set
        self.k_sharpen=params.map_modification.k_sharpen
        self.optimize_b_blur_hires=params.map_modification.optimize_b_blur_hires
        self.iterate=params.map_modification.iterate
        self.optimize_d_cut=params.map_modification.optimize_d_cut
        self.sharpening_target=params.map_modification.sharpening_target
        self.residual_target=params.map_modification.residual_target
        self.eps=params.map_modification.eps
        self.n_bins=params.map_modification.n_bins
        self.input_weight_map_pickle_file=params.input_files.input_weight_map_pickle_file
        self.output_weight_map_pickle_file=params.output_files.output_weight_map_pickle_file
        self.read_sharpened_maps=params.map_modification.read_sharpened_maps
        self.write_sharpened_maps=params.map_modification.write_sharpened_maps
        self.select_sharpened_map=params.map_modification.select_sharpened_map
        self.output_directory=params.output_files.output_directory
        self.smoothing_radius=params.map_modification.smoothing_radius
        self.local_sharpening=params.map_modification.local_sharpening
        self.local_aniso_in_local_sharpening=\
           params.map_modification.local_aniso_in_local_sharpening
        self.overall_before_local=\
           params.map_modification.overall_before_local
        self.box_in_auto_sharpen=params.map_modification.box_in_auto_sharpen
        self.density_select_in_auto_sharpen=params.map_modification.density_select_in_auto_sharpen
        self.density_select_threshold_in_auto_sharpen=params.map_modification.density_select_threshold_in_auto_sharpen
        self.use_weak_density=params.map_modification.use_weak_density
        self.discard_if_worse=params.map_modification.discard_if_worse
        self.box_center=params.map_modification.box_center
        self.box_size=params.map_modification.box_size
        self.target_n_overlap=params.map_modification.target_n_overlap
        self.restrict_map_size=params.map_modification.restrict_map_size
        self.remove_aniso=params.map_modification.remove_aniso
        self.min_ratio_of_ncs_copy_to_first=\
           params.segmentation.min_ratio_of_ncs_copy_to_first
        self.max_ratio_to_target=params.segmentation.max_ratio_to_target
        self.min_ratio_to_target=params.segmentation.min_ratio_to_target
        self.residues_per_region=params.segmentation.residues_per_region
        self.mask_padding_fraction=\
           params.segmentation.mask_padding_fraction
        self.fraction_of_max_mask_threshold=\
           params.segmentation.fraction_of_max_mask_threshold
        self.cell_cutoff_for_solvent_from_mask=\
           params.segmentation.cell_cutoff_for_solvent_from_mask
        self.starting_density_threshold=\
           params.segmentation.starting_density_threshold
        self.density_threshold=params.segmentation.density_threshold
        self.min_ratio=params.segmentation.min_ratio
        self.min_volume=params.segmentation.min_volume
        self.search_b_min=params.map_modification.search_b_min
        self.search_b_max=params.map_modification.search_b_max
        self.search_b_n=params.map_modification.search_b_n
        self.adjust_region_weight=params.map_modification.adjust_region_weight
        self.region_weight_method=params.map_modification.region_weight_method
        self.region_weight_factor=params.map_modification.region_weight_factor
        self.region_weight_buffer=params.map_modification.region_weight_buffer
        self.region_weight_default=params.map_modification.region_weight_default
        self.target_b_iso_ratio=params.map_modification.target_b_iso_ratio
        self.signal_min=params.map_modification.signal_min
        self.target_b_iso_model_scale=params.map_modification.target_b_iso_model_scale
  
        if sharpening_method is not None:
          self.sharpening_method=sharpening_method
  
        if not self.sharpening_method and \
           len(params.map_modification.auto_sharpen_methods)==1:
          self.sharpening_method=params.map_modification.auto_sharpen_methods[0]
  
        if half_map_data_list or self.sharpening_method=='half_map_sharpening':
          self.sharpening_method='half_map_sharpening'
          self.sharpening_target='half_map'
  
        elif pdb_inp or self.sharpening_method=='model_sharpening':
          self.sharpening_method='model_sharpening'
          self.box_in_auto_sharpen=True
          self.density_select_in_auto_sharpen=False
          self.sharpening_target='model'
  
        elif params.map_modification.b_iso is not None or \
            params.map_modification.b_sharpen is not None:
          if self.sharpening_method is None:
            raise Sorry("b_iso is not set")
          # if sharpening values are specified, set them
          if params.map_modification.b_iso is not None:
            self.b_iso=params.map_modification.b_iso # but we need b_sharpen
          elif params.map_modification.b_sharpen is not None:
            self.b_sharpen=params.map_modification.b_sharpen
        elif (params.map_modification.resolution_dependent_b is not None
          and params.map_modification.resolution_dependent_b!=[0,0,0]):
          self.sharpening_method='resolution_dependent'
          self.resolution_dependent_b=\
              params.map_modification.resolution_dependent_b
  
        if self.sharpening_method=='b_iso' and self.k_sharpen is not None:
          self.k_sharpen=None
        return self
  
    def show_summary(self,verbose=False,out=sys.stdout):
      method_summary_dict={
         'b_iso':"Overall b_iso sharpening",
         'b_iso_to_d_cut':"b_iso sharpening to high_resolution cutoff",
         'resolution_dependent':"Resolution-dependent sharpening",
         'model_sharpening':"Model sharpening",
         'half_map_sharpening':"Half-map sharpening",
         'no_sharpening':"No sharpening",
         None:"No sharpening",
          }
  
      target_summary_dict={
         'adjusted_sa':"Adjusted surface area",
         'kurtosis':"Map kurtosis",
         'model':"Map-model CC",
        }
      print("\nSummary of sharpening:\n", file=out)
  
      print("Sharpening method used:         %s\n" %(
         method_summary_dict.get(self.sharpening_method)), file=out)
  
      if self.sharpening_method=="b_iso":
        if self.b_sharpen is not None:
          print("Overall b_sharpen applied:      %7.2f A**2" %(
            self.b_sharpen), file=out)
        if self.b_iso is not None:
          print("Final b_iso obtained:           %7.2f A**2" %(self.b_iso), file=out)
      elif self.sharpening_method=="b_iso_to_d_cut":
        if self.b_sharpen is not None:
          print("Overall b_sharpen applied:      %7.2f A**2" %(
            self.b_sharpen), file=out)
        if self.b_iso is not None:
          print("Final b_iso obtained:           %7.2f A**2" %(self.b_iso), file=out)
        if self.input_d_cut:
          print("High-resolution cutoff:         %7.2f A" %(self.input_d_cut), file=out)
        else:
          print("High-resolution cutoff:         %7.2f A" %(self.resolution), file=out)
      elif self.sharpening_method=="resolution_dependent":
        print("Resolution-dependent b values (%7.2f,%7.2f,%7.2f)\n" %(
          tuple(self.resolution_dependent_b)), file=out)
  
        print("Effective b_iso vs resolution obtained:", file=out)
        from cctbx.maptbx.refine_sharpening import get_effective_b_values
        d_min_values,b_values=get_effective_b_values(
          d_min_ratio=self.d_min_ratio,
           resolution_dependent_b=self.resolution_dependent_b,
           resolution=self.resolution)
        print("                                Resolution  Effective B-iso", file=out)
        print("                                    (A)         (A**2)", file=out)
        for dd,b in zip(d_min_values,b_values):
          print("                                 %7.1f       %7.2f " %(
           dd,b), file=out)
  
      elif self.sharpening_method=="model_sharpening":
        print("Resolution-dependent model sharpening", file=out)
        if self.d_min_list and self.target_scale_factors:
          print("Scale vs resolution:", file=out)
          for d_min,sc in zip(
            self.d_min_list,
            self.target_scale_factors):
            print("Dmin: %7.2f  Scale: %9.6f" %(d_min,sc), file=out)
  
      elif self.sharpening_method=="half_map_sharpening":
        print("Resolution-dependent half-map sharpening", file=out)
        if self.d_min_list and self.target_scale_factors:
          print("Scale vs resolution:", file=out)
          for d_min,sc in zip(
            self.d_min_list,
            self.target_scale_factors):
            print("Dmin: %7.2f  Scale: %9.6f" %(d_min,sc), file=out)
  
      if self.sharpening_method in ["b_iso_to_d_cut"] and \
        self.k_sharpen and self.resolution:
          print("Transition from sharpening"+\
          " to not sharpening (k_sharpen):%7.2f " %(self.k_sharpen), file=out)
  
      print("\nSharpening target used:         %s" %(
         target_summary_dict.get(self.sharpening_target)), file=out)
      if self.adjusted_sa is not None:
        print("Final adjusted map surface area:  %7.2f" %(self.adjusted_sa), file=out)
      if self.kurtosis is not None:
        print("Final map kurtosis:               %7.2f" %(self.kurtosis), file=out)
  
      print(file=out)
  
      if verbose:
        for x in dir(self):
          if x.startswith("__"): continue
          if type(getattr(self,x)) in [type('a'),type(1),type(1.),type([]),
            type((1,2,))]:
            print("%s : %s" %(x,getattr(self,x)), file=out)
  
    def get_effective_b_iso(self,map_data=None,out=sys.stdout):
      map_coeffs_ra,map_coeffs,f_array,phases=effective_b_iso(
        map_data=map_data,
        resolution=self.resolution,
        d_min_ratio=self.d_min_ratio,
        scale_max=self.scale_max,
        crystal_symmetry=self.crystal_symmetry,
         out=out)
      return map_coeffs_ra.b_iso
  
    def sharpen_and_score_map(self,map_data=None,set_b_iso=False,out=sys.stdout):
      if self.n_real is None: # need to get it
        self.n_real=map_data.all()
      map_and_b=sharpen_map_with_si(
          sharpening_info_obj=self,
          map_data=map_data,
          resolution=self.resolution,out=out)
      self.map_data=map_and_b.map_data
      if set_b_iso:
        self.b_iso=map_and_b.final_b_iso
      score_map(map_data=self.map_data,
          sharpening_info_obj=self,
          out=null_out())
      return self
  
    def show_score(self,out=sys.stdout):
      print("Adjusted surface area: %7.3f  Kurtosis: %7.3f  Score: %7.3f\n" %(
         self.adjusted_sa,self.kurtosis,self.score), file=out)
  
    def is_target_b_iso_to_d_cut(self):
      if self.sharpening_method=='target_b_iso_to_d_cut':
         return True
      else:
         return False
  
    def is_b_iso_sharpening(self):
      if self.is_resolution_dependent_sharpening():
         return False
      if self.is_model_sharpening():
         return False
      if self.is_half_map_sharpening():
         return False
      return True
  
    def is_resolution_dependent_sharpening(self):
      if self.sharpening_method=='resolution_dependent':
         return True
      else:
         return False
  
    def is_model_sharpening(self):
      if self.sharpening_method=='model_sharpening':
         return True
      else:
         return False
  
    def is_half_map_sharpening(self):
      if self.sharpening_method=='half_map_sharpening':
         return True
      else:
         return False
  
    def as_map_coeffs(self,out=sys.stdout):
      map_data=getattr(self,'map_data',None)
      if map_data:
        map_coeffs,dummy=get_f_phases_from_map(map_data=self.map_data,
         crystal_symmetry=self.crystal_symmetry,
         d_min=self.resolution,
         d_min_ratio=self.d_min_ratio,
         scale_max=self.scale_max,
         return_as_map_coeffs=True,
         out=out)
        return map_coeffs
      else:
        return None
  
    def as_map_data(self):
      return getattr(self,'map_data',None)
  
  
  class ncs_group_object:
    def __init__(self,
        ncs_obj=None,
        ncs_ops_used=None,
        ncs_group_list=None,
        edited_mask=None,
        crystal_symmetry=None,
        max_cell_dim=None,
        origin_shift=None,
        edited_volume_list=None,
        region_range_dict=None,
        selected_regions=None,
        ncs_related_regions=None,
        self_and_ncs_related_regions=None,
        equiv_dict=None,
        map_files_written=None,
        bad_region_list=None,
        region_centroid_dict=None,
        region_scattered_points_dict=None,
        shared_group_dict=None,
        co=None,
        min_b=None,
        max_b=None,
        original_id_from_id=None,
        remainder_id_dict=None,  # dict relating regions in a remainder object to
                                 #  those in the original map
           ):
      if not selected_regions: selected_regions=[]
      if not ncs_related_regions: ncs_related_regions=[]
      if not self_and_ncs_related_regions: self_and_ncs_related_regions=[]
      if not map_files_written: map_files_written=[]
      from libtbx import adopt_init_args
      adopt_init_args(self, locals())
  
      if self.crystal_symmetry and not self.max_cell_dim:
        self.max_cell_dim=0.
        for x in self.crystal_symmetry.unit_cell().parameters()[:3]:
          self.max_cell_dim=max(max_cell_dim,x)
  
    def as_info_object(self):
      return info_object(
        ncs_obj=self.ncs_obj,
        max_b=self.max_b,
        min_b=self.min_b,
        ncs_group_list=self.ncs_group_list,
        origin_shift=self.origin_shift,
        edited_volume_list=self.edited_volume_list,
        region_range_dict=self.region_range_dict,
        selected_regions=self.selected_regions,
        ncs_related_regions=self.ncs_related_regions,
        self_and_ncs_related_regions=self.self_and_ncs_related_regions,
        bad_region_list=self.bad_region_list,
        region_centroid_dict=self.region_centroid_dict,
        original_id_from_id=self.original_id_from_id,
        map_files_written=self.map_files_written,
       )
  
    def set_ncs_ops_used(self,ncs_ops_used):
      self.ncs_ops_used=deepcopy(ncs_ops_used)
  
    def set_selected_regions(self,selected_regions):
      self.selected_regions=deepcopy(selected_regions)
  
    def set_ncs_related_regions(self,ncs_related_regions):
      self.ncs_related_regions=deepcopy(ncs_related_regions)
  
    def set_self_and_ncs_related_regions(self,self_and_ncs_related_regions):
      self.self_and_ncs_related_regions=deepcopy(self_and_ncs_related_regions)
  
    def set_map_files_written(self,map_files_written):
      self.map_files_written=deepcopy(map_files_written)
  
  def scale_map(map,scale_rms=1.0,out=sys.stdout):
      sd=map.as_double().as_1d().sample_standard_deviation()
      if (sd > 1.e-10):
        scale=scale_rms/sd
        if 0: print("Scaling map by %7.3f to set SD=1" %(scale), file=out)
        map=map*scale
      else:
        print("Cannot scale map...all zeros", file=out)
      return map
  
  def scale_map_coeffs(map_coeffs,scale_max=None,out=sys.stdout):
    f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
    max_value=f_array.data().min_max_mean().max
    if scale_max:
      scale=scale_max/max(1.e-10,max_value)
    else:
      scale=1.0
    if 0:
      print("Scaling map_coeffs by %9.3f to yield maximum of %7.0f" %(
       scale,scale_max), file=out)
    return f_array.array(data=f_array.data()*scale
         ).phase_transfer(phase_source=phases, deg=True)
  
  
  def get_map_object(file_name=None,must_allow_sharpening=None,
        get_map_labels=None,out=sys.stdout):
    # read a ccp4 map file and return sg,cell and map objects 2012-01-16
    if not os.path.isfile(file_name):
      raise Sorry("The map file %s is missing..." %(file_name))
    map_labels=None
    if file_name.endswith(".xplor"):
      import iotbx.xplor.map
      m = iotbx.xplor.map.reader(file_name=file_name)
      m.unit_cell_grid=m.data.all() # just so we have something
      m.space_group_number=0 # so we have something
    else:
      from iotbx import mrcfile
      m = mrcfile.map_reader(file_name=file_name)
      print("MIN MAX MEAN RMS of map: %7.2f %7.2f  %7.2f  %7.2f " %(
        m.header_min, m.header_max, m.header_mean, m.header_rms), file=out)
      print("grid: ",m.unit_cell_grid, file=out)
      print("cell:  %8.3f %8.3f %8.3f %8.3f %8.3f %8.3f  " %tuple(
         m.unit_cell_parameters), file=out)
      print("SG: ",m.space_group_number, file=out)
      if must_allow_sharpening and m.cannot_be_sharpened():
        raise Sorry("Input map is already modified and should not be sharpened")
      if get_map_labels:
        map_labels=m.labels
    print("ORIGIN: ",m.data.origin(), file=out)
    print("EXTENT: ",m.data.all(), file=out)
    print("IS PADDED: ",m.data.is_padded(), file=out)
  
    map_data=m.data
    acc=map_data.accessor()
    shift_needed = not \
       (map_data.focus_size_1d() > 0 and map_data.nd() == 3 and
        map_data.is_0_based())
    if(shift_needed):
      map_data = map_data.shift_origin()
      origin_shift=(
        m.data.origin()[0]/m.data.all()[0],
        m.data.origin()[1]/m.data.all()[1],
        m.data.origin()[2]/m.data.all()[2])
      origin_frac=origin_shift  # NOTE: fraction of NEW cell
    else:
      origin_frac=(0.,0.,0.)
    # determine if we need to trim off the outer part of the map duplicating inner
    offsets=[]
    need_offset=False
    for g,e in zip(m.unit_cell_grid,map_data.all() ):
      offset=e-g
      offsets.append(offset)
    if  offsets == [1,1,1]:
      if origin_frac!=(0.,0.,0.):  # this was a shifted map...we can't do this
        raise Sorry("Sorry if a CCP4 map has an origin other than (0,0,0) "+
          "the extent \nof the map must be the same as the grid or 1 "+
          "\ngreater for "+
          "segment_and_split_map routines."+
         "The file %s has a grid of %s and extent of %s" %(
         file_name,str(m.unit_cell_grid),str(map_data.all())))
      map=map_data[:-1,:-1,:-1]
      acc=map.accessor()
    else:
      map=map_data
  
    # now get space group and cell
    from cctbx import crystal
    from cctbx import sgtbx
    from cctbx import uctbx
    if m.space_group_number==0:
      n=1 # fix mrc formatting
    else:
      n=m.space_group_number
    if hasattr(m,'unit_cell_parameters'):
      space_group_info=sgtbx.space_group_info(number=n)
      unit_cell=uctbx.unit_cell(m.unit_cell_parameters)
      original_unit_cell_grid=m.unit_cell_grid
      original_crystal_symmetry=crystal.symmetry(
        unit_cell=unit_cell,space_group_info=space_group_info)
      if original_crystal_symmetry and map.all()==m.unit_cell_grid:
        crystal_symmetry=original_crystal_symmetry
        print("\nUnit cell crystal symmetry used: ", file=out)
      else:
        crystal_symmetry=m.crystal_symmetry()
        print("\nBox crystal symmetry used: ", file=out)
      crystal_symmetry.show_summary(f=out)
      space_group=crystal_symmetry.space_group()
      unit_cell=crystal_symmetry.unit_cell()
    else:
      space_group=None
      unit_cell=None
      crystal_symmetry=None
      original_crystal_symmetry,original_unit_cell_grid=None,None
  
    map=scale_map(map,out=out)
    if get_map_labels:
      return map,space_group,unit_cell,crystal_symmetry,origin_frac,acc,\
        original_crystal_symmetry,original_unit_cell_grid,map_labels
    else:
      return map,space_group,unit_cell,crystal_symmetry,origin_frac,acc,\
        original_crystal_symmetry,original_unit_cell_grid
  
  def write_ccp4_map(crystal_symmetry, file_name, map_data,
     output_unit_cell_grid=None, labels=None):
    if output_unit_cell_grid is None:
      output_unit_cell_grid=map_data.all()
    if labels is None:
      labels=flex.std_string([""])
  
    iotbx.mrcfile.write_ccp4_map(
        file_name=file_name,
        unit_cell=crystal_symmetry.unit_cell(),
        space_group=crystal_symmetry.space_group(),
        unit_cell_grid = output_unit_cell_grid,
        map_data=map_data.as_double(),
        labels=labels)
  
  def set_up_xrs(crystal_symmetry=None):  # dummy xrs to write out atoms
  
    lines=["ATOM     92  SG  CYS A  10       8.470  28.863  18.423  1.00 22.05           S"] # just a random line to set up x-ray structure
    from cctbx.array_family import flex
    from cctbx import xray
    pdb_inp=iotbx.pdb.input(source_info="",lines=lines)
    xrs = pdb_inp.xray_structure_simple(crystal_symmetry=crystal_symmetry)
    scatterers = flex.xray_scatterer()
    return xrs,scatterers
  
  def write_atoms(tracking_data=None,sites=None,file_name=None,
        crystal_symmetry=None,
        atom_name=None,resname=None,atom_type=None,occ=None,
        out=sys.stdout):
      if crystal_symmetry is None:
         crystal_symmetry=tracking_data.crystal_symmetry
      xrs,scatterers=set_up_xrs(crystal_symmetry=crystal_symmetry)
      from cctbx import xray
      unit_cell=crystal_symmetry.unit_cell()
      for xyz_cart in sites:
        scatterers.append( xray.scatterer(scattering_type="O",
           label="O",
          site=unit_cell.fractionalize(xyz_cart), u=0.38, occupancy=1.0))
      text=write_xrs(xrs=xrs,scatterers=scatterers,file_name=file_name,out=out)
      if atom_name and resname and atom_type:
        text=text.replace("O      O  "," %2s  %3s A" %(atom_name,resname) )
        text=text.replace("           O","           %1s" %(atom_type))
      if occ:
        text=text.replace(" 1.00 "," %.2f " %(occ))
      return text
  
  
  def write_xrs(xrs=None,scatterers=None,file_name="atoms.pdb",out=sys.stdout):
    from cctbx import xray
    xrs = xray.structure(xrs, scatterers=scatterers)
    text=xrs.as_pdb_file()
    if file_name:
      f=open(file_name,'w')
      print(text, file=f)
      f.close()
      print("Atoms written to %s" %file_name, file=out)
    return text
  
  def get_b_iso(miller_array,d_min=None,return_aniso_scale_and_b=False,
      d_max=100000.):
  
    if d_min:
      res_cut_array=miller_array.resolution_filter(d_max=d_max,
         d_min=d_min)
    else:
      res_cut_array=miller_array
  
    from mmtbx.scaling import absolute_scaling
    try:
      aniso_scale_and_b=absolute_scaling.ml_aniso_absolute_scaling(
        miller_array=res_cut_array, n_residues=200, n_bases=0, ignore_errors=True)
      b_cart=aniso_scale_and_b.b_cart
    except Exception as e:
      b_cart=[0,0,0]
      aniso_scale_and_b=None
    b_aniso_mean=0.
    if b_cart:
      for k in [0,1,2]:
        b_aniso_mean+=b_cart[k]
    if return_aniso_scale_and_b:
      return b_aniso_mean/3.0,aniso_scale_and_b
    else: # usual
      return b_aniso_mean/3.0
  
  def map_coeffs_as_fp_phi(map_coeffs):
    amplitudes=map_coeffs.amplitudes()
    amplitudes.set_observation_type_xray_amplitude()
    assert amplitudes.is_real_array()
    phases=map_coeffs.phases(deg=True)
    return amplitudes,phases
  
  def map_coeffs_to_fp(map_coeffs):
    amplitudes=map_coeffs.amplitudes()
    amplitudes.set_observation_type_xray_amplitude()
    assert amplitudes.is_real_array()
    return amplitudes
  
  def get_f_phases_from_model(f_array=None,pdb_inp=None,overall_b=None,
       k_sol=None, b_sol=None, out=sys.stdout):
    xray_structure=pdb_inp.construct_hierarchy().extract_xray_structure(
       crystal_symmetry=f_array.crystal_symmetry())
    print("Getting map coeffs from model with %s atoms.." %(
      xray_structure.sites_frac().size()), file=out)
  
  
    if overall_b is not None:
      print("Setting overall b_iso to %7.1f for model " %(
        overall_b), file=out)
      xray_structure.set_b_iso(value=overall_b)
    model_f_array=f_array.structure_factors_from_scatterers(
        xray_structure = xray_structure).f_calc()
  
    return model_f_array
  
  def get_f_phases_from_map(map_data=None,crystal_symmetry=None,d_min=None,
        d_max=100000.,
        d_min_ratio=None,return_as_map_coeffs=False,remove_aniso=None,
        get_remove_aniso_object=True,
        scale_max=None,
        origin_frac=None,
          out=sys.stdout):
  
      if d_min is not None:
        d_min_use=d_min
        if d_min_ratio is not None:
          d_min_use=d_min*d_min_ratio
      else:
        d_min_use=None
      from mmtbx.command_line.map_to_structure_factors import run as map_to_sf
      if crystal_symmetry.space_group().type().number() in [0,1]:
        args=['d_min=None','box=True','keep_origin=False',
           'scale_max=%s' %scale_max]
      else: # cannot use box for other space groups
        args=['d_min=%s'%(d_min_use),'box=False','keep_origin=False',
           'scale_max=%s' %scale_max]
      map_coeffs=map_to_sf(args=args,
           space_group_number=crystal_symmetry.space_group().type().number(),
           ccp4_map=make_ccp4_map(map_data,crystal_symmetry.unit_cell()),
           return_as_miller_arrays=True,nohl=True,out=null_out())
      if d_min_use:
        map_coeffs=map_coeffs.resolution_filter(d_min=d_min_use,d_max=d_max)
  
      if origin_frac and tuple(origin_frac) != (0.,0.,0.):  # shift origin
        map_coeffs=map_coeffs.translational_shift(origin_frac,deg=False)
  
      map_coeffs=scale_map_coeffs(map_coeffs,scale_max=scale_max,out=out)
  
      if remove_aniso:
        print("\nRemoving aniso in data before analysis\n", file=out)
        get_remove_aniso_object=True
  
      from cctbx.maptbx.refine_sharpening import analyze_aniso
      map_coeffs,map_coeffs_ra=analyze_aniso(
           remove_aniso=remove_aniso,
           get_remove_aniso_object=get_remove_aniso_object,
           map_coeffs=map_coeffs,resolution=d_min,out=out)
  
      if return_as_map_coeffs:
        return map_coeffs,map_coeffs_ra
      else:
        return map_coeffs_as_fp_phi(map_coeffs)
  
  
  def apply_sharpening(map_coeffs=None,
      sharpening_info_obj=None,
      n_real=None,b_sharpen=None,crystal_symmetry=None,
      target_scale_factors=None,
      f_array=None,phases=None,d_min=None,k_sharpen=None,
      b_blur_hires=None,
      out=sys.stdout):
      if map_coeffs and f_array is None and phases is None:
        f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
  
      if sharpening_info_obj is not None:
        b_sharpen=sharpening_info_obj.b_sharpen
        b_blur_hires=sharpening_info_obj.b_blur_hires
        k_sharpen=sharpening_info_obj.k_sharpen
        if sharpening_info_obj.input_d_cut:
          d_min=sharpening_info_obj.input_d_cut
        else:
          d_min=sharpening_info_obj.resolution# changed from d_cut
        n_real=sharpening_info_obj.n_real
        target_scale_factors=sharpening_info_obj.target_scale_factors
        n_bins=sharpening_info_obj.n_bins
        remove_aniso=sharpening_info_obj.remove_aniso
        resolution=sharpening_info_obj.resolution
  
      if target_scale_factors:
        assert sharpening_info_obj is not None
        print("\nApplying target scale factors vs resolution", file=out)
        if not map_coeffs:
          map_coeffs=f_array.phase_transfer(phase_source=phases,deg=True)
        f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
        f_array_b_iso=get_b_iso(f_array,d_min=d_min)
        if not f_array.binner():
          (local_d_max,local_d_min)=f_array.d_max_min()
          f_array.setup_binner(n_bins=n_bins,d_max=local_d_max,d_min=local_d_min)
  
        from cctbx.maptbx.refine_sharpening import apply_target_scale_factors
        map_and_b=apply_target_scale_factors(f_array=f_array,phases=phases,
          resolution=d_min,
          target_scale_factors=target_scale_factors,
          n_real=n_real,
          out=out)
        return map_and_b
  
      elif b_sharpen is None or (
          b_sharpen in [0,None] and k_sharpen in [0,None]):
        if not map_coeffs:
          map_coeffs=f_array.phase_transfer(phase_source=phases,deg=True)
        map_data=get_map_from_map_coeffs(map_coeffs=map_coeffs,
          crystal_symmetry=crystal_symmetry,n_real=n_real)
        return map_and_b_object(map_data=map_data)
  
      elif k_sharpen is None or d_min is None or k_sharpen<=0 or \
          ( b_blur_hires is None and b_sharpen < 0):
        # 2016-08-10 original method: apply b_sharpen to all data
        # Use this if blurring (b_sharpen<0) or if k_sharpen is not set
        from cctbx import adptbx # next lines from xtriage (basic_analysis.py)
        b_cart_aniso_removed=[ b_sharpen, b_sharpen, b_sharpen, 0, 0, 0]
        from mmtbx.scaling import absolute_scaling
        u_star_aniso_removed=adptbx.u_cart_as_u_star(
          f_array.unit_cell(), adptbx.b_as_u( b_cart_aniso_removed  ) )
        f_array_sharpened=absolute_scaling.anisotropic_correction(
          f_array,0.0,u_star_aniso_removed,must_be_greater_than=-0.0001)
      else:
        # Apply sharpening only to data from infinity to d_min, with transition
        # steepness of k_sharpen.
        # 2017-08-21 if b_blur_hires is set, sharpen with
        # b_sharpen-b_blur_hires data beyond d_min (with same
        # transition, so transition goes from b_sharpen TO b_sharpen-b_blur_hires
        data_array=f_array.data()
        sthol_array=f_array.sin_theta_over_lambda_sq()
        d_spacings=f_array.d_spacings()
        scale_array=flex.double()
        import math
  
        if b_blur_hires is not None:
          b_sharpen_hires_use=b_sharpen-b_blur_hires
        else:
          b_sharpen_hires_use=0.
        for x,(ind,sthol),(ind1,d) in zip(data_array,sthol_array,d_spacings):
          # for small value b=b_sharpen
          # for large value b=-b_sharpen_hires_use
          # transition is determined by k_sharpen
          value=min(20.,max(-20.,k_sharpen*(d_min-d)))
          lowres_weight=1./(1.+math.exp(value))
          hires_weight=max(0.,1-lowres_weight)
          b_sharpen_use=b_sharpen*lowres_weight+b_sharpen_hires_use*hires_weight
          log_scale=sthol*b_sharpen_use
          scale_array.append(math.exp(log_scale))
        data_array=data_array*scale_array
        f_array_sharpened=f_array.customized_copy(data=data_array)
  
      actual_b_iso=get_b_iso(f_array_sharpened,d_min=d_min)
      print("B-iso after sharpening by b_sharpen=%6.1f is %7.2f\n" %(
        b_sharpen,actual_b_iso), file=out)
      sharpened_map_coeffs=f_array_sharpened.phase_transfer(
        phase_source=phases,deg=True)
  
      # And get new map
      map_data=get_map_from_map_coeffs(map_coeffs=sharpened_map_coeffs,
        crystal_symmetry=crystal_symmetry,
         n_real=n_real)
      mb=map_and_b_object(map_data=map_data,final_b_iso=actual_b_iso)
      return mb
  
  def get_map_from_map_coeffs(map_coeffs=None,crystal_symmetry=None,
       n_real=None,apply_sigma_scaling=True):
      from cctbx import maptbx
      from cctbx.maptbx import crystal_gridding
      if map_coeffs.crystal_symmetry().space_group_info()!= \
         crystal_symmetry.space_group_info():
        assert str(map_coeffs.crystal_symmetry().space_group_info()
            ).replace(" ","").lower()=='p1'
        # use map_coeffs.crystal_symmetry
        crystal_symmetry=map_coeffs.crystal_symmetry()
      if n_real:
        cg=crystal_gridding(
          unit_cell=crystal_symmetry.unit_cell(),
          space_group_info=crystal_symmetry.space_group_info(),
          pre_determined_n_real=n_real)
      else:
        cg=None
      fft_map = map_coeffs.fft_map( resolution_factor = 0.25,
         crystal_gridding=cg,
         symmetry_flags=maptbx.use_space_group_symmetry)
      if apply_sigma_scaling:
        fft_map.apply_sigma_scaling()
      else:
        fft_map.apply_volume_scaling()
      map_data=fft_map.real_map_unpadded()
      return map_data
  
  def find_symmetry_center(map_data,crystal_symmetry=None,out=sys.stdout):
    # find center if necessary:
    origin=list(map_data.origin())
    all=list(map_data.all())
    centroid_wx={}
    centroid_w={}
    from cctbx import maptbx
    for ai in [0,1,2]:
      centroid_wx[ai]=0.
      centroid_w[ai]=0.
      for i in range(0,all[ai]):
        if ai==0:
          start_tuple=tuple((i,0,0))
          end_tuple=tuple((i,all[1],all[2]))
        elif ai==1:
           start_tuple=tuple((0,i,0))
           end_tuple=tuple((all[0],i,all[2]))
        elif ai==2:
           start_tuple=tuple((0,0,i))
           end_tuple=tuple((all[0],all[1],i))
        new_map_data = maptbx.copy(map_data,
           start_tuple,end_tuple)
        mean_value=max(0.,new_map_data.as_1d().as_double().min_max_mean().mean)
        centroid_wx[ai]+=mean_value*(i-origin[ai])
        centroid_w[ai]+=mean_value
      if centroid_w[ai]>0:
        centroid_wx[ai]=centroid_wx[ai]/centroid_w[ai]
    print("CENTROID OF DENSITY: (%7.2f, %7.2f, %7.2f) (grid units) " %(
      tuple((centroid_wx[0],centroid_wx[1],centroid_wx[2],))), file=out)
    xyz_fract=matrix.col((centroid_wx[0]/all[0],centroid_wx[1]/all[1],centroid_wx[2]/all[2],))
    xyz_cart=crystal_symmetry.unit_cell().orthogonalize(xyz_fract)
    print("CENTROID (A): (%7.3f, %7.3f, %7.3f) " %(
      tuple(xyz_cart)), file=out)
    return xyz_cart
  
  def get_center_of_map(map_data,crystal_symmetry):
    all=list(map_data.all())
    origin=list(map_data.origin())
    sx,sy,sz=[all[0]/2+origin[0],all[1]/2+origin[1],all[2]/2+origin[2]]
    site_fract=matrix.col((sx/all[0],sy/all[1],sz/all[2],))
    return crystal_symmetry.unit_cell().orthogonalize(site_fract)
  
  
  def select_remaining_ncs_ops( map_data=None,
      crystal_symmetry=None,
      random_points=None,
      closest_sites=None,
      ncs_object=None,
      out=sys.stdout):
  
    # identify which NCS ops still apply.  Choose the ones that maximize
    # scoring with score_ncs_in_map
    if ncs_object.max_operators()<1:
      return ncs_object
  
    used_ncs_id_list=[ncs_object.ncs_groups()[0].identity_op_id()]
    ncs_copies=ncs_object.max_operators()
  
    # find ncs_id that maximizes score (if any)
    improving=True
    from copy import deepcopy
    best_ops_to_keep=deepcopy(used_ncs_id_list)
    working_best_ops_to_keep=None
    best_score=None
  
    while improving:
      improving=False
      working_best_ops_to_keep=deepcopy(best_ops_to_keep)
      working_score=None
      for ncs_id in range(ncs_copies):
        if ncs_id in best_ops_to_keep:continue
        ops_to_keep=deepcopy(best_ops_to_keep)
        ops_to_keep.append(ncs_id)
        ncs_used_obj=ncs_object.deep_copy(ops_to_keep=ops_to_keep)
        score,ncs_cc=score_ncs_in_map(map_data=map_data,ncs_object=ncs_used_obj,
          ncs_in_cell_only=True,
          allow_score_with_pg=False,
          sites_orth=closest_sites,
          crystal_symmetry=crystal_symmetry,out=null_out())
        if score is None: continue
        if working_score is None or score >working_score:
          working_score=score
          working_best_ops_to_keep=deepcopy(ops_to_keep)
      if working_score is not None and (
          best_score is None or working_score>best_score):
        improving=True
        best_score=working_score
        best_ops_to_keep=deepcopy(working_best_ops_to_keep)
  
    ncs_used_obj=ncs_object.deep_copy(ops_to_keep=best_ops_to_keep)
    return ncs_used_obj
  
  def run_get_ncs_from_map(params=None,
        map_data=None,
        crystal_symmetry=None,
        map_symmetry_center=None,
        ncs_obj=None,
        out=sys.stdout,
        ):
  
    # Get or check NCS operators. Try various possibilities for center of NCS
    ncs_obj_to_check=None
    if params.reconstruction_symmetry.symmetry and (
       not ncs_obj or ncs_obj.max_operators()<2):
      if params.reconstruction_symmetry.optimize_center:
        center_try_list=[True,False]
      else:
        center_try_list=[True]
    elif ncs_obj:
      center_try_list=[True]
      ncs_obj_to_check=ncs_obj
    elif params.reconstruction_symmetry.optimize_center:
      center_try_list=[None]
    else:
      return None,None,None # did not even try
    # check separately for helical symmetry
    if params.reconstruction_symmetry.symmetry.lower()=='helical':
      helical_list=[True]
    elif params.reconstruction_symmetry.symmetry.lower() in ['all','any'] and\
        params.reconstruction_symmetry.include_helical_symmetry:
      helical_list=[False,True]
    else:
      helical_list=[False]
  
    new_ncs_obj,ncs_cc,ncs_score=None,None,None
    for use_center_of_map in center_try_list:
     for include_helical in helical_list:
      local_params=deepcopy(params)
      local_params.reconstruction_symmetry.include_helical_symmetry=\
         include_helical
      new_ncs_obj,ncs_cc,ncs_score=get_ncs_from_map(params=local_params,
        map_data=map_data,
        map_symmetry_center=map_symmetry_center,
        use_center_of_map_as_center=use_center_of_map,
        crystal_symmetry=crystal_symmetry,
        ncs_obj_to_check=ncs_obj_to_check,
        out=out
        )
      if new_ncs_obj:
        return new_ncs_obj,ncs_cc,ncs_score
    return new_ncs_obj,ncs_cc,ncs_score
  
  def get_ncs_from_map(params=None,
        map_data=None,
        map_symmetry_center=None,
        symmetry=None,
        symmetry_center=None,
        helical_rot_deg=None,
        helical_trans_z_angstrom=None,
        two_fold_along_x=None,
        op_max=None,
        crystal_symmetry=None,
        optimize_center=None,
        sites_orth=None,
        random_points=None,
        n_rescore=None,
        use_center_of_map_as_center=None,
        min_ncs_cc=None,
        identify_ncs_id=None,
        ncs_obj_to_check=None,
        ncs_in_cell_only=False,
        out=sys.stdout):
  
  
  # Purpose: check through standard point groups and helical symmetry to see
    # if map has symmetry. If symmetry==ANY then take highest symmetry that fits
    # Otherwise limit to the one specified with symmetry.
    #  Use a library of symmetry matrices.  For helical symmetry generate it
    #  along the z axis.
    # Center of symmetry is as supplied, or center of map or center of density
    #  If center is not supplied and use_center_of_map_as_center, try that
    #  and return None if it fails to achieve a map cc of min_ncs_cc
  
    if ncs_in_cell_only is None:
      ncs_in_cell_only=(not params.crystal_info.use_sg_symmetry)
    if symmetry is None:
      symmetry=params.reconstruction_symmetry.symmetry
    if symmetry_center is None:
      symmetry_center=params.reconstruction_symmetry.symmetry_center
    if optimize_center is None:
      optimize_center=params.reconstruction_symmetry.optimize_center
    if helical_rot_deg is None:
      helical_rot_deg=params.reconstruction_symmetry.helical_rot_deg
    if helical_trans_z_angstrom is None:
      helical_trans_z_angstrom=\
        params.reconstruction_symmetry.helical_trans_z_angstrom
    if n_rescore is None:
      n_rescore=params.reconstruction_symmetry.n_rescore
    if random_points is None:
      random_points=params.reconstruction_symmetry.random_points
    if op_max is None:
      op_max=params.reconstruction_symmetry.op_max
    if two_fold_along_x is None:
      two_fold_along_x=params.reconstruction_symmetry.two_fold_along_x
    if identify_ncs_id is None:
      identify_ncs_id=params.reconstruction_symmetry.identify_ncs_id
    if min_ncs_cc is None:
      min_ncs_cc=params.reconstruction_symmetry.min_ncs_cc
  
    # if ncs_obj_to_check is supplied...just use that ncs
    if ncs_obj_to_check and ncs_obj_to_check.max_operators()>1:
      symmetry="SUPPLIED NCS"
  
  
    if map_symmetry_center is None:
      map_symmetry_center=get_center_of_map(map_data,crystal_symmetry)
  
    if optimize_center is None:
      if symmetry_center is None and (not use_center_of_map_as_center):
        optimize_center=True
        print("Setting optimize_center=True as no symmetry_center is supplied", file=out)
      else:
        optimize_center=False
  
    if symmetry_center is not None:
      symmetry_center=matrix.col(symmetry_center)
    elif use_center_of_map_as_center:
      print("Using center of map as NCS center", file=out)
      symmetry_center=map_symmetry_center
    else: # Find it
      if not ncs_obj_to_check:
        print("Finding NCS center as it is not supplied", file=out)
      symmetry_center=find_symmetry_center(
      map_data,crystal_symmetry=crystal_symmetry,
         out=out)
    print("Center of NCS (A): (%7.3f, %7.3f, %7.3f) " %(
      tuple(symmetry_center)), file=out)
  
    ncs_list=get_ncs_list(params=params,
      symmetry=symmetry,
     symmetry_center=symmetry_center,
     helical_rot_deg=helical_rot_deg,
     two_fold_along_x=two_fold_along_x,
     op_max=op_max,
     helical_trans_z_angstrom=helical_trans_z_angstrom,
     ncs_obj_to_check=ncs_obj_to_check,
     map_data=map_data,
     crystal_symmetry=crystal_symmetry,
     out=out,
     )
  
    print("Total of %d NCS types to examine..." %(len(ncs_list)), file=out)
    if not sites_orth:
      sites_orth=get_points_in_map(
       map_data,n=random_points,crystal_symmetry=crystal_symmetry)
    # some random points in the map
  
    # Now make sure symmetry applied to points in points_list gives similar values
  
    results_list=[]
    for ncs_obj in ncs_list:
      symmetry=ncs_obj.get_ncs_name()
      score,cc_avg=score_ncs_in_map(map_data=map_data,ncs_object=ncs_obj,
         identify_ncs_id=identify_ncs_id,
         ncs_in_cell_only=ncs_in_cell_only,
        sites_orth=sites_orth,crystal_symmetry=crystal_symmetry,out=out)
      if cc_avg < min_ncs_cc:
        score=0. # Do not allow low CC values to be used
      if score is None:
        print("symmetry:",symmetry," no score",ncs_obj.max_operators(), file=out)
      else:
        results_list.append([score,cc_avg,ncs_obj,symmetry])
    if not results_list:
      return None,None,None
  
    results_list.sort()
    results_list.reverse()
  
    # Rescore top n_rescore
    if n_rescore and not ncs_obj_to_check:
      print("Rescoring top %d results" %(min(n_rescore,len(results_list))), file=out)
      rescore_list=results_list[n_rescore:]
      new_sites_orth=get_points_in_map(
        map_data,n=10*random_points,crystal_symmetry=crystal_symmetry)
      new_sites_orth.extend(sites_orth)
      for orig_score,orig_cc_avg,ncs_obj,symmetry in results_list[:n_rescore]:
        score,cc_avg=score_ncs_in_map(map_data=map_data,ncs_object=ncs_obj,
          identify_ncs_id=identify_ncs_id,
          ncs_in_cell_only=ncs_in_cell_only,
          sites_orth=new_sites_orth,crystal_symmetry=crystal_symmetry,out=out)
        if cc_avg < min_ncs_cc:
          score=0. # Do not allow low CC values to be used
        if score is None:
          print("symmetry:",symmetry," no score",ncs_obj.max_operators(), file=out)
        else:
          rescore_list.append([score,cc_avg,ncs_obj,symmetry])
      rescore_list.sort()
      rescore_list.reverse()
      results_list=rescore_list
    if len(results_list)==1:
      # check for C1
      score,cc_avg,ncs_obj,symmetry=results_list[0]
      if symmetry and symmetry.strip()=='C1':
        score=1.
        cc_avg=1.
        results_list=[[score,cc_avg,ncs_obj,symmetry],]
  
  
    print("Ranking of NCS types:", file=out)
    if min_ncs_cc is not None:
      print("NOTE: any NCS type with CC < %.2f (min_ncs_cc) is unscored " %(
        min_ncs_cc), file=out)
    print("\n  SCORE    CC   OPERATORS     SYMMETRY", file=out)
    for score,cc_avg,ncs_obj,symmetry in results_list:
      if not symmetry: symmetry=""
      if not cc_avg: cc_avg=0.0
  
  
      print(" %6.2f  %5.2f    %2d          %s" %(
         score,cc_avg,ncs_obj.max_operators(), symmetry.strip(),), file=out)
  
    score,cc_avg,ncs_obj,ncs_info=results_list[0]
  
    # Optimize center if necessary
    if optimize_center:
      symmetry_center,cc_avg,score,ncs_obj=optimize_center_position(
         map_data,sites_orth,
         crystal_symmetry,
         ncs_info,symmetry_center,ncs_obj,score,cc_avg,
         params=params,
         helical_rot_deg=helical_rot_deg,
         two_fold_along_x=two_fold_along_x,
         op_max=op_max,
         min_ncs_cc=min_ncs_cc,
         identify_ncs_id=identify_ncs_id,
         ncs_obj_to_check=ncs_obj_to_check,
         ncs_in_cell_only=ncs_in_cell_only,
         helical_trans_z_angstrom=helical_trans_z_angstrom,out=out)
      print("New center: (%7.3f, %7.3f, %7.3f)" %(tuple(symmetry_center)), file=out)
  
    if cc_avg < min_ncs_cc:
      print("No suitable symmetry found", file=out)
      return None,None,None
  
    print("\nBest NCS type is: ", end=' ', file=out)
    print("\n  SCORE    CC   OPERATORS     SYMMETRY", file=out)
    if not ncs_info: ncs_info=""
    print(" %6.2f  %5.2f    %2d          %s  Best NCS type" %(
         score,cc_avg,ncs_obj.max_operators(), ncs_info.strip(),), file=out)
    return ncs_obj,cc_avg,score
  
  
  def optimize_center_position(map_data,sites_orth,crystal_symmetry,
       ncs_info,symmetry_center,ncs_obj,score,cc_avg,
       params=None,
       helical_rot_deg=None,
       two_fold_along_x=None,
       op_max=None,
       identify_ncs_id=None,
       ncs_obj_to_check=None,
       ncs_in_cell_only=None,
       min_ncs_cc=None,
       helical_trans_z_angstrom=None,out=sys.stdout):
  
    if ncs_info is None:
      ncs_info="None"
    symmetry=ncs_info.split()[0]
    print("Optimizing center position...type is %s" %(ncs_info), file=out)
  
    if len(ncs_info.split())>1 and ncs_info.split()[1]=='(a)':
      two_fold_along_x=True
    elif len(ncs_info.split())>1 and ncs_info.split()[1]=='(b)':
      two_fold_along_x=False
    else:
      two_fold_along_x=None
  
    best_center=matrix.col(symmetry_center)
    best_ncs_obj=ncs_obj
    best_score=score
    best_cc_avg=cc_avg
    print("Starting center: (%7.3f, %7.3f, %7.3f)" %(tuple(best_center)), file=out)
    from libtbx.utils import null_out
    scale=5.
    for itry in range(6):
      scale=scale/5.
      for i in range(-4,5):
       for j in range(-4,5):
        local_center=matrix.col(symmetry_center)+matrix.col((scale*i,scale*j,0.,))
        ncs_list=get_ncs_list(params=params,symmetry=symmetry,
         symmetry_center=local_center,
         helical_rot_deg=helical_rot_deg,
         two_fold_along_x=two_fold_along_x,
         op_max=op_max,
         helical_trans_z_angstrom=helical_trans_z_angstrom,
         ncs_obj_to_check=ncs_obj_to_check,
         map_data=map_data,
         crystal_symmetry=crystal_symmetry,
         out=null_out(),
         )
        if ncs_list:
          ncs_obj=ncs_list[0]
          score,cc_avg=score_ncs_in_map(map_data=map_data,ncs_object=ncs_obj,
            identify_ncs_id=identify_ncs_id,
            ncs_in_cell_only=ncs_in_cell_only,
            sites_orth=sites_orth,crystal_symmetry=crystal_symmetry,out=out)
          if cc_avg < min_ncs_cc:
            score=0. # Do not allow low CC values to be used
        else:
          ncs_obj=None
          score,cc_avg=None,None
        if best_score is None or score>best_score:
          best_cc_avg=cc_avg
          best_score=score
          best_center=local_center
          best_ncs_obj=ncs_obj
  
    symmetry_center=best_center
    cc_avg=best_cc_avg
    score=best_score
    ncs_obj=best_ncs_obj
    return best_center,best_cc_avg,best_score,best_ncs_obj
  
  
  def score_ncs_in_map_point_group_symmetry(
      map_data=None,ncs_object=None,sites_orth=None,
       crystal_symmetry=None,out=sys.stdout):
    ncs_group=ncs_object.ncs_groups()[0]
    all_value_lists=[]
    for c,t,r in zip(ncs_group.centers(),
                         ncs_group.translations_orth(),
                         ncs_group.rota_matrices()):
      new_sites_cart=flex.vec3_double()
      r_inv=r.inverse()
      for site in sites_orth:
        new_sites_cart.append(r_inv * (matrix.col(site) - t))
      # get value at new_sites cart and make sure they are all the same...
      new_sites_fract=crystal_symmetry.unit_cell().fractionalize(new_sites_cart)
      values=flex.double()
      for site_fract in new_sites_fract:
        values.append(map_data.value_at_closest_grid_point(site_fract))
      all_value_lists.append(values)
    return get_cc_among_value_lists(all_value_lists)
  
  def get_cc_among_value_lists(all_value_lists):
    a=all_value_lists[0]
    cc_avg=0.
    cc_low=None
    cc_n=0.
    for j in range(1,len(all_value_lists)):
        b=all_value_lists[j]
        cc=flex.linear_correlation(a,b).coefficient()
        cc_avg+=cc
        cc_n+=1.
        if cc_low is None or cc<cc_low:
          cc_low=cc
    cc_avg=cc_avg/max(1.,cc_n)
    if cc_n>0:
      import math
      return cc_low*math.sqrt(len(all_value_lists)),cc_avg
    else:
      return None,None
  
  def score_ncs_in_map(map_data=None,ncs_object=None,sites_orth=None,
       identify_ncs_id=None,
       ncs_in_cell_only=None,
       allow_score_with_pg=True,
       crystal_symmetry=None,out=sys.stdout):
    if not ncs_object or ncs_object.max_operators()<2:
      return None,None
  
    ncs_group=ncs_object.ncs_groups()[0]
        # don't use point-group symmetry if we have only some of the ops
    if allow_score_with_pg and (
       (not identify_ncs_id) or ncs_group.is_point_group_symmetry()):
      return score_ncs_in_map_point_group_symmetry(
       map_data=map_data,ncs_object=ncs_object,
       sites_orth=sites_orth,crystal_symmetry=crystal_symmetry,out=out)
  
          # This version does not assume point-group symmetry: find the NCS
          #  operator that maps each point on to all others the best, then save
    #  that list of values
  
    identify_ncs_id_list=list(range(ncs_group.n_ncs_oper()))+[None]
  
    all_value_lists=[]
  
    if not sites_orth:
      sites_orth=get_points_in_map(map_data,n=100,
      minimum_fraction_of_max=0.05,
      crystal_symmetry=crystal_symmetry)
  
    for site in sites_orth:
      best_id=0
      best_score=None
      best_values=None
      for site_ncs_id in identify_ncs_id_list:  #last is real one
        if site_ncs_id is None:
          site_ncs_id=best_id
          real_thing=True
        else:
          real_thing=False
  
        if identify_ncs_id and site_ncs_id:
          local_site=ncs_group.rota_matrices()[site_ncs_id] * matrix.col(site) + \
             ncs_group.translations_orth()[site_ncs_id]
        else:
          local_site=site
        new_sites_cart=flex.vec3_double()
        for c,t,r in zip(ncs_group.centers(),
                         ncs_group.translations_orth(),
                         ncs_group.rota_matrices()):
          r_inv=r.inverse()
          new_sites_cart.append(r_inv * (matrix.col(local_site) - t))
        new_sites_fract=crystal_symmetry.unit_cell().fractionalize(
           new_sites_cart)
        values=flex.double()
        for site_frac in new_sites_fract:
          if (not ncs_in_cell_only) or (
                site_frac[0]>=0 and site_frac[0]<=1 and  \
                site_frac[1]>=0 and site_frac[1]<=1 and  \
                site_frac[2]>=0 and site_frac[2]<=1):
            values.append(map_data.value_at_closest_grid_point(site_frac))
          else:
            values.append(0.)
  
        score=values.standard_deviation_of_the_sample()
        if real_thing or (best_score is None or score < best_score):
            best_score=score
            best_id=site_ncs_id
            best_values=values
      all_value_lists.append(best_values)
  
    values_by_site_dict={} # all_value_lists[j][i] -> values_by_site_dict[i][j]
    # there are sites_orth.size() values of j
    # there are len(ncs_group_centers)==len(all_value_lists[0]) values of i
    for i in range(len(all_value_lists[0])):
      values_by_site_dict[i]=flex.double() # value_list[0][1]
      for j in range(sites_orth.size()):
        values_by_site_dict[i].append(all_value_lists[j][i])
    new_all_values_lists=[]
    for i in range(len(all_value_lists[0])):
      new_all_values_lists.append(values_by_site_dict[i])
    score,cc=get_cc_among_value_lists(new_all_values_lists)
    return score,cc
  
  def get_points_in_map(map_data,n=None,
        minimum_fraction_of_max=0.,
        random_xyz=None,
        max_tries_ratio=100,crystal_symmetry=None):
    map_1d=map_data.as_1d()
    map_mean=map_1d.min_max_mean().mean
    map_max=map_1d.min_max_mean().max
    minimum_value=map_mean+minimum_fraction_of_max*(map_max-map_mean)
    points_list=flex.vec3_double()
    import random
    random.seed(1)
  
    nu,nv,nw=map_data.all()
    xyz_fract=crystal_symmetry.unit_cell().fractionalize(
         tuple((17.4,27.40128571,27.32985714,)))
    for i in range(int(max_tries_ratio*n)): # max tries
      ix=random.randint(0,nu-1)
      iy=random.randint(0,nv-1)
      iz=random.randint(0,nw-1)
      xyz_fract=matrix.col((ix/nu,iy/nv,iz/nw,))
      value=map_data.value_at_closest_grid_point(xyz_fract)
      if value > minimum_value and value <map_max:
        if random_xyz:
           offset=[]
           for i in range(3):
             offset.append((random.random()-0.5)*2.*random_xyz)
           offset=crystal_symmetry.unit_cell().fractionalize(matrix.col(offset))
           new_xyz_fract=[]
           for x,o in zip(xyz_fract,offset):
             new_xyz_fract.append(max(0,min(1,x+o)))
           xyz_fract=matrix.col(new_xyz_fract)
        points_list.append(xyz_fract)
        if points_list.size()>=n: break
    sites_orth=crystal_symmetry.unit_cell().orthogonalize(points_list)
    return sites_orth
  
  
  
  def get_ncs_list(params=None,symmetry=None,
     symmetry_center=None,
     helical_rot_deg=None,
     helical_trans_z_angstrom=None,
     op_max=None,
     two_fold_along_x=None,
     ncs_obj_to_check=None,
     crystal_symmetry=None,
     map_data=None,
     include_helical_symmetry=None,
     max_helical_ops_to_check=None,
     require_helical_or_point_group_symmetry=None,
      out=sys.stdout):
    # params.reconstruction_symmetry.require_helical_or_point_group_symmetry
    # params.reconstruction_symmetry.include_helical_symmetry):
    # params.reconstruction_symmetry.max_helical_ops_to_check))
  
    if ncs_obj_to_check and ncs_obj_to_check.max_operators()>1:
      return [ncs_obj_to_check] # ,["SUPPLIED NCS"]
  
    from mmtbx.ncs.ncs import generate_ncs_ops
    ncs_list=generate_ncs_ops(
     symmetry=symmetry,
     helical_rot_deg=helical_rot_deg,
     helical_trans_z_angstrom=helical_trans_z_angstrom,
     op_max=op_max,
     two_fold_along_x=two_fold_along_x,
     include_helical_symmetry=\
         params.reconstruction_symmetry.include_helical_symmetry,
     max_helical_ops_to_check=\
         params.reconstruction_symmetry.max_helical_ops_to_check,
     require_helical_or_point_group_symmetry=\
        params.reconstruction_symmetry.require_helical_or_point_group_symmetry,
     out=out)
  
    # Generate helical symmetry from map if necessary
    if symmetry.lower()=='helical' or (
        symmetry.lower() in ['all','any'] and
        params.reconstruction_symmetry.include_helical_symmetry):
       if helical_rot_deg is None or helical_trans_z_angstrom is None:
  
       # returns ncs for symmetry_center at symmetry_center
        ncs_object,helical_rot_deg,helical_trans_z_angstrom=\
           find_helical_symmetry(params=params,
          symmetry_center=symmetry_center,
          map_data=map_data,
          crystal_symmetry=crystal_symmetry,out=out)
  
        if ncs_object:
          ncs_name="Type: Helical %5.2f deg  %6.2f Z-trans " %(
            helical_rot_deg,helical_trans_z_angstrom)
          ncs_object.set_ncs_name(ncs_name)
          ncs_list.append(ncs_object)
  
    if symmetry_center and tuple(symmetry_center) != (0,0,0,):
      print("Offsetting NCS center by (%.2f, %.2f, %.2f) A " %(tuple(symmetry_center)), file=out)
      new_list=[]
      for ncs_obj in ncs_list:
        new_list.append(ncs_obj.coordinate_offset(coordinate_offset=symmetry_center))
      ncs_list=new_list
  
    if (require_helical_or_point_group_symmetry):
      for ncs_obj in ncs_list:
        assert ncs_obj.is_helical_along_z() or ncs_obj.is_point_group_symmetry()
  
    return ncs_list
  
  def find_helical_symmetry(params=None,
          symmetry_center=None,
          map_data=None,
          crystal_symmetry=None,
          max_z_to_test=2, max_peaks_to_score=5,out=sys.stdout):
  
    params=deepcopy(params) # so changing them does not go back
    if str(params.reconstruction_symmetry.score_basis)=='None':
      params.reconstruction_symmetry.score_basis='cc'
    if params.reconstruction_symmetry.smallest_object is None:
      params.reconstruction_symmetry.smallest_object=\
        5*params.crystal_info.resolution
  
    print("\nLooking for helical symmetry with center at (%.2f, %.2f, %.2f) A\n" %(
        tuple(symmetry_center)), file=out)
    print("\nFinding likely translations along z...", file=out)
  
    map_coeffs,dummy=get_f_phases_from_map(map_data=map_data,
         crystal_symmetry=crystal_symmetry,
         d_min=params.crystal_info.resolution,
         return_as_map_coeffs=True,
         out=out)
    f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
    from cctbx.maptbx.refine_sharpening import quasi_normalize_structure_factors
    (d_max,d_min)=f_array.d_max_min()
    f_array.setup_binner(d_max=d_max,d_min=d_min,n_bins=20)
    normalized=quasi_normalize_structure_factors(
            f_array,set_to_minimum=0.01)
  
    # Now look along c* and get peaks
    zero_index_a=0  # look along c*
    zero_index_b=1
    c_star_list=get_c_star_list(f_array=normalized,
       zero_index_a=zero_index_a,
       zero_index_b=zero_index_b)
  
    likely_z_translations=get_helical_trans_z_angstrom(params=params,
       c_star_list=c_star_list,
      crystal_symmetry=crystal_symmetry,
      out=out)
  
    # Now for each z...get the rotation. Try +/- z and try rotations
    abc=crystal_symmetry.unit_cell().parameters()[:3]
    max_z=max(abc[0],abc[1])
    if params.reconstruction_symmetry.max_helical_rotations_to_check:
      min_delta_rot=360/max(1,
      params.reconstruction_symmetry.max_helical_rotations_to_check)
    else:
      min_delta_rot=0.01
  
    delta_rot=max(min_delta_rot,
       (180./3.141659)*params.crystal_info.resolution/max_z)
  
    delta_z=params.crystal_info.resolution/4.
    print("\nOptimizing helical paramers:", file=out)
    print("\n Rot   Trans  Score    CC", file=out)
  
    n_rotations=int(0.5+360/delta_rot)
    rotations=[]
    for k in range(1,n_rotations):
      helical_rot_deg=k*delta_rot
      if helical_rot_deg > 180: helical_rot_deg=helical_rot_deg-360
      rotations.append(helical_rot_deg)
  
    done=False
    n_try=0
  
    score_list=[]
    quick=params.control.quick
    best_ncs_cc=None
    best_ncs_obj=None
    best_score=None
    best_helical_trans_z_angstrom=None
    best_helical_rot_deg=None
    import math
    for helical_trans_z_angstrom  in likely_z_translations[:max_z_to_test]:
      n_try+=1
      if done: break
      for helical_rot_deg in rotations:
        if done: break
        if abs(helical_trans_z_angstrom)+abs(math.sin(
          helical_rot_deg*(3.14159/180.))*max_z) < \
           params.reconstruction_symmetry.smallest_object:
           continue  # this is identity
        new_ncs_obj,ncs_cc,ncs_score,\
            new_helical_trans_z_angstrom,new_helical_rot_deg=try_helical_params(
          optimize=0,
          helical_rot_deg=helical_rot_deg,
          helical_trans_z_angstrom=helical_trans_z_angstrom,
          params=params,
          map_data=map_data,
          map_symmetry_center=symmetry_center, # should not be needed XXX
          symmetry_center=symmetry_center,
          crystal_symmetry=crystal_symmetry,
          out=null_out())
  
        if not ncs_score: continue
        if ncs_cc> 0.95 or \
          ncs_cc > 2*params.reconstruction_symmetry.min_ncs_cc or \
          (quick and (ncs_cc> 0.90 or
           ncs_cc > 1.5*params.reconstruction_symmetry.min_ncs_cc)):
          print(" %.2f   %.2f   %.2f   %.2f (ok to go on)" %(
             new_helical_rot_deg,new_helical_trans_z_angstrom,
             ncs_score,ncs_cc), file=out)
          done=True
        if params.control.verbose:
          print(" %.2f   %.2f   %.2f   %.2f" %(
            new_helical_rot_deg,new_helical_trans_z_angstrom,ncs_score,ncs_cc), file=out)
        score_list.append(
         [ncs_score,ncs_cc,new_helical_rot_deg,new_helical_trans_z_angstrom])
      score_list.sort()
      score_list.reverse()
      done=False
      for ncs_score,ncs_cc,helical_rot_deg,helical_trans_z_angstrom in \
         score_list[:max_peaks_to_score]:
        if done: continue
        # rescore and optimize:
  
        new_ncs_obj,ncs_cc,ncs_score,\
            new_helical_trans_z_angstrom,new_helical_rot_deg=try_helical_params(
          params=params,
          best_score=best_score,
          helical_rot_deg=helical_rot_deg,
          helical_trans_z_angstrom=helical_trans_z_angstrom,
          delta_z=delta_z/2.,
          delta_rot=delta_rot/2.,
          map_data=map_data,
          map_symmetry_center=symmetry_center,
          symmetry_center=symmetry_center,
          crystal_symmetry=crystal_symmetry,
          out=null_out())
        if not ncs_score or ncs_score <0:
          continue
        if best_score is None or ncs_score>best_score:
          best_ncs_cc=ncs_cc
          best_ncs_obj=new_ncs_obj
          best_score=ncs_score
          best_helical_trans_z_angstrom=new_helical_trans_z_angstrom
          best_helical_rot_deg=new_helical_rot_deg
  
  
        # after trying out a range quit if good enough
        if best_ncs_cc> 0.90 or \
            best_ncs_cc > 1.5*params.reconstruction_symmetry.min_ncs_cc or \
            ( (quick or n_try>1) and \
               ncs_cc>=params.reconstruction_symmetry.min_ncs_cc):
          print(" %.2f   %.2f   %.2f   %.2f (high enough to go on)" %(
             best_helical_rot_deg,best_helical_trans_z_angstrom,
             best_score,best_ncs_cc), file=out)
  
  
          done=True
  
    # Optimize one more time trying fractional values, but only if
    #  that makes the delta less than the resolution
  
    print("\nTrying fraction of rot/trans", file=out)
    for iter in [0,1]:
      if not best_helical_rot_deg: continue
      for ifract in range(2,11):
        if iter==0:  # try fractional
          test_helical_rot_deg=best_helical_rot_deg/ifract
          test_helical_trans_z_angstrom=best_helical_trans_z_angstrom/ifract
          if test_helical_trans_z_angstrom > params.crystal_info.resolution*1.1:
            continue # skip it...would have been a peak if ok
        else: # iter >0
          if ifract > 0:
             continue # skip these
          else: # optimize current best
            test_helical_rot_deg=best_helical_rot_deg
            test_helical_trans_z_angstrom=best_helical_trans_z_angstrom
        new_ncs_obj,new_ncs_cc,new_ncs_score,\
              new_helical_trans_z_angstrom,new_helical_rot_deg=\
          try_helical_params(
            params=params,
            helical_rot_deg=test_helical_rot_deg,
            helical_trans_z_angstrom=test_helical_trans_z_angstrom,
            delta_z=delta_z/2.,
            delta_rot=delta_rot/2.,
            map_data=map_data,
            map_symmetry_center=symmetry_center,
            symmetry_center=symmetry_center,
            crystal_symmetry=crystal_symmetry,
            out=out)
        if new_ncs_score is not None:
          new_ncs_score=new_ncs_score*\
             params.reconstruction_symmetry.scale_weight_fractional_translation
             # give slight weight to smaller
  
          if best_score is None or new_ncs_score > best_score:
            best_ncs_cc=new_ncs_cc
            best_ncs_obj=new_ncs_obj
            best_score=new_ncs_score
            best_helical_trans_z_angstrom=new_helical_trans_z_angstrom
            best_helical_rot_deg=new_helical_rot_deg
            print(" %.2f   %.2f   %.2f   %.2f (improved fractions)" %(
               best_helical_rot_deg,best_helical_trans_z_angstrom,
               best_score,best_ncs_cc), file=out)
          else:
            print(" %.2f   %.2f   %.2f   %.2f (worse with fractions)" %(
               new_helical_rot_deg,new_helical_trans_z_angstrom,
               new_ncs_score,new_ncs_cc), file=out)
  
  
    # Optimize one more time trying multiples of values to get better param
    imult=int(0.5+
       0.33*max_z/params.reconstruction_symmetry.max_helical_ops_to_check)
  
  
    working_ncs_cc=best_ncs_cc
    working_ncs_obj=best_ncs_obj
    working_score=None
    if best_helical_rot_deg:
      working_helical_rot_deg=best_helical_rot_deg*imult
      working_helical_trans_z_angstrom=best_helical_trans_z_angstrom*imult
    else:
      working_helical_rot_deg=None
      working_helical_trans_z_angstrom=None
      imult=0
  
    if imult > 1:
      print("\nTrying %sx multiples of rot/trans" %(imult), file=out)
      improved=False
      for iter in [1,2,3]:
        if iter > 1 and not improved: break
        improved=False
  
        new_ncs_obj,new_ncs_cc,new_ncs_score,\
              new_helical_trans_z_angstrom,new_helical_rot_deg=\
          try_helical_params(
            params=params,
            helical_rot_deg=working_helical_rot_deg,
            helical_trans_z_angstrom=working_helical_trans_z_angstrom,
            delta_z=delta_z/(2.*iter),
            delta_rot=delta_rot/(2.*iter),
            map_data=map_data,
            map_symmetry_center=symmetry_center,
            symmetry_center=symmetry_center,
            crystal_symmetry=crystal_symmetry,
            out=out)
        if new_ncs_score is not None:
          if working_score is None or new_ncs_score > working_score:
            working_ncs_cc=new_ncs_cc
            working_ncs_obj=new_ncs_obj
            working_score=new_ncs_score
            working_helical_trans_z_angstrom=new_helical_trans_z_angstrom
            working_helical_rot_deg=new_helical_rot_deg
            print(" %.2f   %.2f   %.2f   %.2f (Scoring for multiple)" %(
                 working_helical_rot_deg,working_helical_trans_z_angstrom,
                 working_score,working_ncs_cc), file=out)
  
        #  and rescore with this:
        working_helical_rot_deg=working_helical_rot_deg/imult
        working_helical_trans_z_angstrom=working_helical_trans_z_angstrom/imult
        for iter in [1,2,3]:
          new_ncs_obj,new_ncs_cc,new_ncs_score,\
              new_helical_trans_z_angstrom,new_helical_rot_deg=\
          try_helical_params(
            params=params,
            helical_rot_deg=working_helical_rot_deg,
            helical_trans_z_angstrom=working_helical_trans_z_angstrom,
            delta_z=delta_z/(2.*iter),
            delta_rot=delta_rot/(2.*iter),
            map_data=map_data,
            map_symmetry_center=symmetry_center,
            symmetry_center=symmetry_center,
            crystal_symmetry=crystal_symmetry,
            out=null_out())
          if new_ncs_score is not None:
            working_ncs_obj,working_ncs_cc,working_ncs_score,\
              working_helical_trans_z_angstrom,working_helical_rot_deg=\
              new_ncs_obj,new_ncs_cc,new_ncs_score,\
              new_helical_trans_z_angstrom,new_helical_rot_deg
            if best_score is None or working_ncs_score > best_score:
              if params.control.verbose:
                print("\nTaking parameters from multiples", file=out)
              best_ncs_cc=working_ncs_cc
              best_ncs_obj=working_ncs_obj
              best_score=working_ncs_score
              best_helical_trans_z_angstrom=working_helical_trans_z_angstrom
              best_helical_rot_deg=working_helical_rot_deg
              print(" %.2f   %.2f   %.2f   %.2f (improved by multiples)" %(
                 best_helical_rot_deg,best_helical_trans_z_angstrom,
                 best_score,best_ncs_cc), file=out)
              improved=True
  
              working_ncs_cc=best_ncs_cc
              working_ncs_obj=best_ncs_obj
              working_score=None
              working_helical_trans_z_angstrom=best_helical_trans_z_angstrom*imult
              working_helical_rot_deg=best_helical_rot_deg*imult
  
  
    if best_helical_rot_deg and best_helical_trans_z_angstrom and best_score and best_ncs_cc:
      # Check to make sure there is no overlap
  
      print(" %.2f   %.2f   %.2f   %.2f (Final)" %(
       best_helical_rot_deg,best_helical_trans_z_angstrom,\
           best_score,best_ncs_cc), file=out)
  
      from mmtbx.ncs.ncs import get_helical_symmetry
  
      ncs_object=get_helical_symmetry(
         helical_rot_deg=best_helical_rot_deg,
         helical_trans_z_angstrom=best_helical_trans_z_angstrom,
         max_ops=params.reconstruction_symmetry.max_helical_ops_to_check)
    else:
      ncs_object=None
  
    return ncs_object,best_helical_rot_deg,best_helical_trans_z_angstrom
  
  def try_helical_params(
      optimize=None,
      best_score=None,
      delta_z=None,
      delta_rot=None,
      helical_rot_deg=None,
      helical_trans_z_angstrom=None,
      params=None,
      map_data=None,
      map_symmetry_center=None,
      symmetry_center=None,
      crystal_symmetry=None,
      out=sys.stdout):
  
    if delta_z is None or delta_rot is None:
      assert not optimize
    local_params=deepcopy(params)
    local_params.reconstruction_symmetry.min_ncs_cc=-100
    local_params.reconstruction_symmetry.n_rescore=0
    local_params.reconstruction_symmetry.symmetry='helical'
    local_params.reconstruction_symmetry.helical_rot_deg=helical_rot_deg
    local_params.reconstruction_symmetry.helical_trans_z_angstrom=\
       helical_trans_z_angstrom
    abc=crystal_symmetry.unit_cell().parameters()[:3]
    max_z=max(abc[0],abc[1])
    import math
    if abs(helical_trans_z_angstrom)+abs(math.sin(
          helical_rot_deg*(3.14159/180.))*max_z) < \
           params.reconstruction_symmetry.smallest_object:
           return None,None,None,\
                  None,None
  
    best_helical_trans_z_angstrom,best_helical_rot_deg=\
       helical_trans_z_angstrom,helical_rot_deg
    best_ncs_score=best_score
    best_ncs_obj=None
    best_ncs_cc=None
  
    test_ncs_obj,test_ncs_cc,test_ncs_score=get_ncs_from_map(params=local_params,
      map_data=map_data,
      map_symmetry_center=symmetry_center,
      symmetry_center=symmetry_center,
      crystal_symmetry=crystal_symmetry,
      out=null_out())
    if params.reconstruction_symmetry.score_basis=='cc':
       test_ncs_score=test_ncs_cc
  
    if best_ncs_score is None or test_ncs_score>best_ncs_score:
      best_ncs_score=test_ncs_score
      best_ncs_cc=test_ncs_cc
      best_ncs_obj=test_ncs_obj
  
    if optimize is None:
      optimize=params.reconstruction_symmetry.max_helical_optimizations
  
  
    # save in case we need to go back
    working_helical_trans_z_angstrom,working_helical_rot_deg=\
       helical_trans_z_angstrom,helical_rot_deg
    working_ncs_score=best_ncs_score
    working_ncs_cc=best_ncs_cc
    working_ncs_obj=best_ncs_obj
  
  
    if optimize and (best_score is None or best_ncs_score > best_score):
      # try with few to many operators..
      if params.control.verbose:
        print("\nOptimizing by varying number of operators", file=out)
        print("Starting score: %.2f" %(working_ncs_score), file=out)
      for k in range(optimize):
       local_params.reconstruction_symmetry.max_helical_ops_to_check=min(k+1,
        params.reconstruction_symmetry.max_helical_ops_to_check)
       best_score=None # start over for each number of operators
       for i in [0,-1,1]:
         for j in [0,-1,1]:
          new_ncs_obj,new_ncs_cc,new_ncs_score,\
              new_helical_trans_z_angstrom,new_helical_rot_deg=try_helical_params(
            optimize=False,
            helical_rot_deg=max(0.1,best_helical_rot_deg+i*delta_rot),
            helical_trans_z_angstrom=max(0.01,best_helical_trans_z_angstrom+j*delta_z),
            delta_z=delta_z,
            delta_rot=delta_rot,
            params=params,
            map_data=map_data,
            map_symmetry_center=symmetry_center,
            symmetry_center=symmetry_center,
            crystal_symmetry=crystal_symmetry,
            out=out)
          if new_ncs_score and new_ncs_score>0 and (
              best_score is None or new_ncs_score>best_score):
            if params.control.verbose:
              print("Working score for %s ops: %.2f" %(
                local_params.reconstruction_symmetry.max_helical_ops_to_check,
                new_ncs_score), file=out)
            best_score=new_ncs_score
            best_ncs_score=new_ncs_score
            best_helical_trans_z_angstrom=new_helical_trans_z_angstrom
            best_helical_rot_deg=new_helical_rot_deg
            best_ncs_obj=new_ncs_obj
            best_ncs_cc=new_ncs_cc
       delta_rot=delta_rot/2
       delta_z=delta_z/2
  
  
      #rescore with what we now have (best values) and compare with working
      local_params.reconstruction_symmetry.max_helical_ops_to_check=\
         params.reconstruction_symmetry.max_helical_ops_to_check
      if params.control.verbose:
        print("Rescoring with original number of operators (%s)" %(
          local_params.reconstruction_symmetry.max_helical_ops_to_check), file=out)
      local_params.reconstruction_symmetry.helical_rot_deg=best_helical_rot_deg
      local_params.reconstruction_symmetry.helical_trans_z_angstrom=\
        best_helical_trans_z_angstrom
      best_ncs_obj,best_ncs_cc,best_ncs_score=get_ncs_from_map(
           params=local_params,
        map_data=map_data,
        map_symmetry_center=symmetry_center,
        symmetry_center=symmetry_center,
        crystal_symmetry=crystal_symmetry,
        out=null_out())
      if params.reconstruction_symmetry.score_basis=='cc':
        best_ncs_score=best_ncs_cc
  
      # now take it if better
      if best_ncs_cc and best_ncs_cc>working_ncs_cc:
        if params.control.verbose:
          print("Using optimized values (%.2f > %.2f)" %(
           best_ncs_cc,working_ncs_cc), file=out)
        # keep these (best)
      else:
        if params.control.verbose:
          print("Rejecting optimized values (%.2f <= %.2f)" %(
           best_ncs_cc,working_ncs_cc), file=out)
  
        # resture working values
        best_helical_trans_z_angstrom,best_helical_rot_deg=\
           working_helical_trans_z_angstrom,working_helical_rot_deg
        best_ncs_score=working_ncs_score
        best_ncs_obj=working_ncs_obj
        best_ncs_cc=working_ncs_cc
  
    if params.reconstruction_symmetry.score_basis=='cc':
       best_ncs_score=best_ncs_cc
  
    return best_ncs_obj,best_ncs_cc,best_ncs_score,\
              best_helical_trans_z_angstrom,best_helical_rot_deg
  
  
  def get_d_and_value_list(c_star_list):
    d_list=[]
    from scitbx.array_family import flex
    value_list=flex.double()
    for hkl,value,d in c_star_list:
      d_list.append(d)
      value_list.append(value)
    max_value=value_list.min_max_mean().max
    if value_list.size()>3:
      max_value=value_list[2]
    new_d_list=[]
    new_value_list=[]
    for d,value in zip(d_list,value_list):
      if value < max_value/1000: # reject those that are really zero
         continue
      new_d_list.append(d)
      new_value_list.append(value)
    return new_d_list,new_value_list
  
  def get_helical_trans_z_angstrom(params=None,
    c_star_list=None,crystal_symmetry=None,
    minimum_ratio=2.,
    max_first_peak=1,out=sys.stdout):
  
    # Find highest-resolution peak along c*...guess it is n*z_translation
    #  where n is small
  
    max_z=flex.double(crystal_symmetry.unit_cell().parameters()[:3]).min_max_mean().max
  
    if params.control.verbose:
      print("Values along c*: ", file=out)
    d_list,value_list=get_d_and_value_list(c_star_list)
  
    for d,value in zip(d_list,value_list):
      if params.control.verbose:
        print(" %.2f A : %.2f " %(d,value), file=out)
  
    d_list,value_list=get_max_min_list(
       d_list=d_list,value_list=value_list,minimum_ratio=1.0)
  
    d_list,value_list=get_max_min_list(
       d_list=d_list,value_list=value_list,minimum_ratio=2.0,
      maximum_only=True)
    sort_list=[]
    for d,value in zip(d_list,value_list):
      sort_list.append([value,d])
    sort_list.sort()
    sort_list.reverse()
    likely_z_translations=[]
    dis_min=params.crystal_info.resolution/4
    for value,d in sort_list:
      likely_z_translations_local=[]
      for i in range(1,max_first_peak+1):
        z=d/i
        if z > max_z: continue
        if z < dis_min: continue # no way
        if is_close_to_any(z=z,others=likely_z_translations,
          dis_min=dis_min): continue
  
        likely_z_translations_local.append(z)
        likely_z_translations.append(z)
  
    print("\nMaximal values along c* and likely z-translations: ", file=out)
    for z in likely_z_translations:
      print(" %.2f A " %(z), end=' ', file=out)
    print(file=out)
    return likely_z_translations
  
  def small_deltas(z_translations,dis_min=None):
    delta_list=[]
    for z,z1 in zip(z_translations,z_translations[1:]):
       delta=abs(z-z1)
       if not is_close_to_any(z=delta,others=z_translations+delta_list,
          dis_min=dis_min):
         delta_list.append(abs(delta))
    return delta_list
  
  def is_close_to_any(z=None,others=None,
      dis_min=None):
    for x in others:
      if abs(x-z)<dis_min:
         return True
    return False
  
  def get_max_min_list(d_list=None,value_list=None,
     minimum_ratio=None,maximum_only=False):
  
    max_min_list=[]
    max_min_d_list=[]
    for value_prev,d_prev, \
         value,d, \
         value_next,d_next in zip(
           value_list+[0,0],
           d_list+[0,0],
           [0]+value_list+[0],
           [0]+d_list+[0],
           [0,0]+value_list,
           [0,0]+d_list):
  
      if d and ( value >=  value_prev *minimum_ratio) and (
         value >= value_next*minimum_ratio):  # local max
        max_min_list.append(value)
        max_min_d_list.append(d)
      if (not maximum_only) and d and ( value <=  value_prev ) and (
         value <= value_next):  # local min
        max_min_list.append(value)
        max_min_d_list.append(d)
    return max_min_d_list,max_min_list
  
  def get_c_star_list(f_array=None,
     zero_index_a=0,zero_index_b=1,zero_index_c=2):
    c_star_list=[]
    for value,(indices,d) in zip(f_array.data(),
       f_array.d_spacings()):
      if indices[zero_index_a]==0 and indices[zero_index_b]==0 and \
        indices[zero_index_c] >=4:
        c_star_list.append([tuple(indices),value,d])
    c_star_list.sort()
    return c_star_list
  
  def get_params_from_args(args):
    command_line = iotbx.phil.process_command_line_with_files(
      map_file_def="input_files.map_file",
      seq_file_def="input_files.seq_file",
      pdb_file_def="input_files.pdb_in",
      ncs_file_def="input_files.ncs_file",
      args=args,
      master_phil=master_phil)
  
    return command_line.work.extract()
  
  
  def get_mask_around_molecule(map_data=None,
          wang_radius=None,
          buffer_radius=None,
          return_masked_fraction=False,
          minimum_fraction_of_max=0.01,
          solvent_content=None,
          solvent_content_iterations=None,
          crystal_symmetry=None, out=sys.stdout):
    # use iterated solvent fraction tool to identify mask around molecule
    try:
      from phenix.autosol.map_to_model import iterated_solvent_fraction
      solvent_fraction,mask=iterated_solvent_fraction(
        crystal_symmetry=crystal_symmetry,
        wang_radius=wang_radius,
        solvent_content=solvent_content,
        solvent_content_iterations=solvent_content_iterations,
        map_as_double=map_data,
        out=out)
    except Exception as e:
      print("No mask obtained...", file=out)
      return None,None
  
    if not mask:
      print("No mask obtained...", file=out)
      return None,None
  
    # Now expand the mask to increase molecular region
    expand_size=estimate_expand_size(
         crystal_symmetry=crystal_symmetry,
         map_data=map_data,
         expand_target=buffer_radius,
         out=out)
  
    print("Target mask expand size is %d based on buffer_radius of %7.1f A" %(
       expand_size,buffer_radius), file=out)
  
    co,sorted_by_volume,min_b,max_b=get_co(map_data=mask,
       threshold=0.5,wrapping=False)
    masked_fraction=sorted_by_volume[1][0]/mask.size()
  
    bool_region_mask = co.expand_mask(id_to_expand=sorted_by_volume[1][1],
         expand_size=expand_size)
    s=(bool_region_mask==True)
    expanded_fraction=s.count(True)/s.size()
    print("\nLargest masked region before buffering: %7.2f" %(masked_fraction),
        file=out)
    print("\nLargest masked region after buffering: %7.2f" %(expanded_fraction),
       file=out)
    if solvent_content:
      delta_as_is=abs(solvent_content- (1-masked_fraction))
      delta_expanded=abs(solvent_content- (1-expanded_fraction))
      if delta_expanded > delta_as_is:
        # already there
        expand_size=0
        print ("Setting expand size to zero as masked fraction already ",
           "close to solvent_content",file=out)
  
    s=None
    minimum_size=sorted_by_volume[1][0] * minimum_fraction_of_max
    if expand_size==0:
      result=co.result()
    else:
      result=None
  
    for v1,i1 in sorted_by_volume[1:]:
      if v1 < minimum_size: break
      if expand_size > 0:
        bool_region_mask = co.expand_mask(
          id_to_expand=i1, expand_size=expand_size)
      else:
        bool_region_mask=(result==i1)
      if s is None:
        s = (bool_region_mask==True)
      else:
        s |= (bool_region_mask==True)
    mask.set_selected(s,1)
    mask.set_selected(~s,0)
    masked_fraction=mask.count(1)/mask.size()
    print("Masked fraction after buffering:  %7.2f" %(masked_fraction), file=out)
    if return_masked_fraction:
      return mask.as_double(),1-masked_fraction
    else: # usual return solvent fraction estimate
      return mask.as_double(),solvent_fraction  # This is solvent fraction est.
  
  def get_mean_in_and_out(sel=None,
      map_data=None,
      verbose=False,
      out=sys.stdout):
  
    mean_value_in,fraction_in=get_mean_in_or_out(sel=sel,
      map_data=map_data,
      out=out)
  
    mean_value_out,fraction_out=get_mean_in_or_out(sel= ~sel,
      map_data=map_data,
      out=out)
  
    if mean_value_out is None:
      mean_value_out=mean_value_in
    if mean_value_in is None:
      mean_value_in=mean_value_out
    if verbose:
      print("\nMean inside mask: %7.2f  Outside mask: %7.2f  Fraction in: %7.2f" %(
       mean_value_in,mean_value_out,fraction_in), file=out)
    return mean_value_in,mean_value_out,fraction_in
  
  def get_mean_in_or_out(sel=None,
      map_data=None,
      out=sys.stdout):
    masked_map=map_data.deep_copy()
    masked_map.set_selected(~sel,0)
    mean_after_zeroing_in_or_out=masked_map.as_1d().min_max_mean().mean
    masked_map.set_selected(sel,1)
    fraction_in_or_out=masked_map.as_1d().min_max_mean().mean
    if fraction_in_or_out >1.e-10:
      mean_value=mean_after_zeroing_in_or_out/fraction_in_or_out
    else:
      mean_value=None
  
    return mean_value,fraction_in_or_out
  
  def apply_soft_mask(map_data=None,
            mask_data=None,
            rad_smooth=None,
            crystal_symmetry=None,
            set_outside_to_mean_inside=False,
            set_mean_to_zero=False,
            threshold=0.5,
            verbose=False,
            out=sys.stdout):
  
    # apply a soft mask based on mask_data to map_data.
    # set value outside mask==mean value inside mask or mean value outside mask
  
  
    s = mask_data > threshold  # s marks inside mask
  
    # get mean inside or outside mask
    if verbose:
      print("\nStarting map values inside and outside mask:", file=out)
    mean_value_in,mean_value_out,fraction_in=get_mean_in_and_out(sel=s,
      verbose=verbose,map_data=map_data, out=out)
  
    if verbose:
      print("\nMask inside and outside values", file=out)
    mask_mean_value_in,mask_mean_value_out,mask_fraction_in=get_mean_in_and_out(
        sel=s,map_data=mask_data, verbose=verbose,out=out)
  
    # Smooth the mask in place. First make it a binary mask
    mask_data = mask_data.set_selected(~s, 0)  # outside mask==0
    mask_data = mask_data.set_selected( s, 1)
    if mask_data.count(1)  and mask_data.count(0): # something to do
      if verbose:
        print("Smoothing mask...", file=out)
      maptbx.unpad_in_place(map=mask_data)
      mask_data = maptbx.smooth_map(
        map              = mask_data,
        crystal_symmetry = crystal_symmetry,
        rad_smooth       = rad_smooth)
  
      # Make sure that mask_data max value is now 1, scale if not
      max_mask_data_value=mask_data.as_1d().min_max_mean().max
      if max_mask_data_value > 1.e-30 and max_mask_data_value!=1.0:
        mask_data=mask_data*(1./max_mask_data_value)
        if verbose:
          print("Scaling mask by %.2f to yield maximum of 1.0 " %(
             1./max_mask_data_value), file=out)
    else:
      if verbose:
        print("Not smoothing mask that is a constant...", file=out)
  
    if verbose:
      print("\nSmoothed mask inside and outside values", file=out)
    smoothed_mean_value_in,smoothed_mean_value_out,smoothed_fraction_in=\
       get_mean_in_and_out(sel=s,map_data=mask_data, verbose=verbose,out=out)
  
    # Now replace value outside mask with mean_value, value inside with current,
    #   smoothly going from one to the other based on mask_data
  
    # set_to_mean will be a constant map with value equal to inside or outside
    if set_outside_to_mean_inside or mean_value_out is None:
      target_value_for_outside=mean_value_in
      if verbose:
        print("Setting value outside mask to mean inside (%.2f)" %(
        target_value_for_outside), file=out)
    else:
      target_value_for_outside=mean_value_out
      if verbose:
        print("Setting value outside mask to mean outside (%.2f)" %(
          target_value_for_outside), file=out)
    set_to_mean=mask_data.deep_copy()
    ss = set_to_mean > -1.e+30 # select everything
    set_to_mean.set_selected(ss, target_value_for_outside)
  
    masked_map= (map_data * mask_data ) +  (set_to_mean * (1-mask_data))
  
    if set_mean_to_zero:  # remove average
      masked_map=masked_map - masked_map.as_1d().min_max_mean().mean
  
    if verbose:
      print("\nFinal mean value inside and outside mask:", file=out)
    mean_value_in,mean_value_out,fraction_in=get_mean_in_and_out(sel=s,
      map_data=masked_map, verbose=verbose,out=out)
  
    return masked_map,mask_data
  
  def estimate_expand_size(
         crystal_symmetry=None,
         map_data=None,
         expand_target=None,
         out=sys.stdout):
      abc = crystal_symmetry.unit_cell().parameters()[:3]
      N_ = map_data.all()
      nn=0.
      for i in range(3):
        delta=abc[i]/N_[i]
        nn+=expand_target/delta
      nn=max(1,int(0.5+nn/3.))
      print("Expand size (grid units): %d (about %4.1f A) " %(
        nn,nn*abc[0]/N_[0]), file=out)
      return max(1,nn)
  
  def get_max_z_range_for_helical_symmetry(params,out=sys.stdout):
    if not params.input_files.ncs_file: return
    ncs_obj,dummy_tracking_data=get_ncs(params=params,out=out)
    if not ncs_obj.is_helical_along_z(): return
    if params.map_modification.restrict_z_distance_for_helical_symmetry:  #take it
       return params.map_modification.restrict_z_distance_for_helical_symmetry
  
    if not params.map_modification.restrict_z_turns_for_helical_symmetry: return
  
    print("Identifying maximum z-range for helical symmetry", file=out)
    print("Maximum of %7.1f turns up and down in Z allowed..." %(
       params.map_modification.restrict_z_turns_for_helical_symmetry), file=out)
    r,t=ncs_obj.ncs_groups()[0].helix_rt_forwards()
    cost=r[0]
    sint=r[1]
    import math
    theta=abs(180.*math.atan2(sint,cost)/3.14159)
    trans=abs(t)
    pitch=trans*360./max(0.1,theta)
    max_z=params.map_modification.restrict_z_turns_for_helical_symmetry*pitch
    print("Z-values restricted to +/- %7.1f A" %(max_z), file=out)
    print("\nRunning map-box once to get position of molecule, again to"+\
        " apply\n Z restriction\n", file=out)
    return max_z
  
  def dist(x,y):
    dd=0.
    for a,b in zip(x,y):
      dd+=(a-b)**2
    return dd**0.5
  
  def get_ncs_closest_sites(
      closest_sites=None,
      sites_cart=None,
      used_ncs_id_list=None,
      box_ncs_object=None,
      box_crystal_symmetry=None,
      out=sys.stdout):
  
    # try to find NCS ops mapping sites_cart close to closest_sites
  
    best_id=None
    best_rms=None
    best_sites=closest_sites.deep_copy()
    for ncs_id in range(box_ncs_object.max_operators()):
      if ncs_id in used_ncs_id_list: continue
  
      test_sites=closest_sites.deep_copy()
      ncs_sites_cart=get_ncs_sites_cart(sites_cart=sites_cart,
         ncs_obj=box_ncs_object,unit_cell=box_crystal_symmetry.unit_cell(),
         ncs_id=ncs_id,
         ncs_in_cell_only=False)
      test_sites.extend(ncs_sites_cart)
      rms=radius_of_gyration_of_vector(test_sites)
      if best_rms is None or rms < best_rms:
        best_rms=rms
        best_ncs_id=ncs_id
        best_sites=test_sites.deep_copy()
    used_ncs_id_list.append(best_ncs_id)
    return best_sites,used_ncs_id_list
  
  def get_closest_sites(
      high_points=None,
      sites_cart=None,
      box_ncs_object=None,
      box_crystal_symmetry=None,
      out=sys.stdout):
  
    if not box_ncs_object.is_point_group_symmetry() and not \
        box_ncs_object.is_helical_along_z():
      # extract point_group symmetry if present and box_ncs_object doesn't have it
      print("Trying to extract point-group symmetry from box_ncs_object "+\
         "with %d ops" %( box_ncs_object.max_operators()), file=out)
  
      ncs_object=box_ncs_object.deep_copy(extract_point_group_symmetry=True)
      if ncs_object:
        print("New number of operators satisfying point-group symmetry: %d" %(
          ncs_object.max_operators()), file=out)
        box_ncs_object=ncs_object
  
      else:
        print("No point-group symmetry found", file=out)
  
  
    ncs_copies=box_ncs_object.max_operators()
    closest_sites=high_points
    from scitbx.matrix import col
    for id in range(sites_cart.size()):
      local_sites_cart=sites_cart[id:id+1]
      local_sites_cart.extend(get_ncs_sites_cart(sites_cart=local_sites_cart,
         ncs_obj=box_ncs_object,unit_cell=box_crystal_symmetry.unit_cell(),
         ncs_in_cell_only=True))
      if local_sites_cart.size() <ncs_copies: continue  # some were out of range
  
      xx=col((0.,0.,0.,))
      for site in closest_sites:
        xx+=col(site)
      xx=xx/max(1,closest_sites.size())
      target=flex.vec3_double()
      target.append(xx)
  
      dd,id1,id2=target.min_distance_between_any_pair_with_id(
         local_sites_cart)
      best_points=local_sites_cart[id2:id2+1]
      closest_sites.extend(best_points)
    return closest_sites[1:]
  
  def get_range(sites=None,unit_cell=None,map_data=None,
     boundary_tolerance=None,out=sys.stdout):
    x_values=flex.double()
    y_values=flex.double()
    z_values=flex.double()
    for site_cart in sites:
      (x,y,z)=tuple(site_cart)
      x_values.append(x)
      y_values.append(y)
      z_values.append(z)
    x_min_max_mean=x_values.min_max_mean()
    x_min=x_min_max_mean.min
    x_max=x_min_max_mean.max
    y_min_max_mean=y_values.min_max_mean()
    y_min=y_min_max_mean.min
    y_max=y_min_max_mean.max
    z_min_max_mean=z_values.min_max_mean()
    z_min=z_min_max_mean.min
    z_max=z_min_max_mean.max
    print("\nRange for box:", file=out)
    print("            X        Y        Z", file=out)
    print(" LOW:  %7.1f    %7.1f     %7.1f " %(tuple([x_min,y_min,z_min])), file=out)
    print(" HIGH: %7.1f    %7.1f     %7.1f \n" %(tuple([x_max,y_max,z_max])), file=out)
  
    # move to 0, 1 if near ends
    if x_min<=boundary_tolerance: x_min=0.
    if y_min<=boundary_tolerance: y_min=0.
    if z_min<=boundary_tolerance: z_min=0.
    a,b,c,al,bet,gam=unit_cell.parameters()
    if x_min>=a-boundary_tolerance: x_min=a
    if y_min>=b-boundary_tolerance: y_min=b
    if z_min>=c-boundary_tolerance: z_min=c
    print("\nAdjusted range for box:", file=out)
    print("            X        Y        Z", file=out)
    print(" LOW:  %7.1f    %7.1f     %7.1f " %(tuple([x_min,y_min,z_min])), file=out)
    print(" HIGH: %7.1f    %7.1f     %7.1f \n" %(tuple([x_max,y_max,z_max])), file=out)
  
    nx,ny,nz=map_data.all()
    # convert to grid units
    i_min=max(0,min(nx,int(0.5+nx*x_min/a)))
    j_min=max(0,min(ny,int(0.5+ny*y_min/b)))
    k_min=max(0,min(nz,int(0.5+nz*z_min/c)))
    i_max=max(0,min(nx,int(0.5+nx*x_max/a)))
    j_max=max(0,min(ny,int(0.5+ny*y_max/b)))
    k_max=max(0,min(nz,int(0.5+nz*z_max/c)))
    lower_bounds=[i_min,j_min,k_min]
    upper_bounds=[i_max,j_max,k_max]
  
    print("\nGrid bounds for box:", file=out)
    print("            X        Y        Z", file=out)
    print(" LOW:  %7d    %7d     %7d " %(tuple([i_min,j_min,k_min])), file=out)
    print(" HIGH: %7d    %7d     %7d \n" %(tuple([i_max,j_max,k_max])), file=out)
  
    return lower_bounds,upper_bounds
  
  def get_bounds_for_au_box(params,
       box=None,out=sys.stdout):
  
    # Try to get bounds for a box that include one au
  
    if not box.ncs_object or box.ncs_object.max_operators()<2:
      return None,None,None
  
    box_ncs_object=box.ncs_object
    box_map_data=box.map_box
    box_crystal_symmetry=box.box_crystal_symmetry
    random_points=10*params.reconstruction_symmetry.random_points
  
    sites_cart=get_points_in_map(box_map_data,n=random_points,
      minimum_fraction_of_max=params.segmentation.density_select_threshold,
      random_xyz=params.crystal_info.resolution*2.,
      crystal_symmetry=box_crystal_symmetry)
    assert sites_cart.size() >0
  
    # apply symmetry to sites_orth and see where the molecule is
    ncs_sites_cart=get_ncs_sites_cart(sites_cart=sites_cart,
         ncs_obj=box_ncs_object,unit_cell=box_crystal_symmetry.unit_cell(),
         ncs_in_cell_only=True)
  
    # generate this in a lower-memory way XXX
    low_res_map_data=get_low_res_map_data(sites_cart=ncs_sites_cart,
      d_min=params.crystal_info.resolution*7.,
      crystal_symmetry=box_crystal_symmetry,
      out=out)
  
    high_points=get_high_points_from_map(  # actually returns just one.
          map_data=low_res_map_data,
          unit_cell=box_crystal_symmetry.unit_cell(),out=out)
    from scitbx.matrix import col
    high_points=high_points[0:1]
    cutout_center=col(high_points[0])
    print("Center of box will be near (%7.1f,%7.1f,%7.1f)" %(
       tuple(cutout_center)))
  
    # now figure out box that contains at least one copy of each ncs-related
    #  point.
  
    # Find closest ncs-related points for each unique random point to this
    # center. Starting box is the box that contains all of these.
  
    closest_sites=get_closest_sites(
      high_points=high_points,
      sites_cart=sites_cart,
      box_ncs_object=box_ncs_object,
      box_crystal_symmetry=box_crystal_symmetry,
      out=out)
    if not closest_sites or closest_sites.size()<1:
      print("\nNo sites representing au of map found...skipping au box\n", file=out)
      return None,None,None
  
    print("\nTotal of %d sites representing 1 au found" %(
        closest_sites.size()), file=out)
  
    # write out closest_sites to match original position
    coordinate_offset=-1*matrix.col(box.shift_cart)
    write_atoms(file_name='one_au.pdb',
        crystal_symmetry=box_crystal_symmetry,sites=closest_sites+coordinate_offset)
  
    unique_closest_sites=closest_sites.deep_copy()
    # Now if desired, find NCS-related groups of sites
    if params.segmentation.n_au_box >1:
      print("\nFinding up to %d related au" %(params.segmentation.n_au_box), file=out)
      print("Starting RMSD of sites: %7.1f A " %(
         radius_of_gyration_of_vector(closest_sites)), file=out)
  
      sites_orig=closest_sites.deep_copy()
      used_ncs_id_list=[box_ncs_object.ncs_groups()[0].identity_op_id()]
      for i in range(params.segmentation.n_au_box-1):
        closest_sites,used_ncs_id_list=get_ncs_closest_sites(
          used_ncs_id_list=used_ncs_id_list,
          closest_sites=closest_sites,
          sites_cart=sites_orig,
          box_ncs_object=box_ncs_object,
          box_crystal_symmetry=box_crystal_symmetry,
          out=out)
      print("\nNew total of %d sites representing %d au found" %(
        closest_sites.size(),params.segmentation.n_au_box), file=out)
      print("New rmsd: %7.1f A " %(
         radius_of_gyration_of_vector(closest_sites)), file=out)
  
    lower_bounds,upper_bounds=get_range(
       sites=closest_sites,map_data=box_map_data,
       boundary_tolerance=params.crystal_info.resolution,
       unit_cell=box_crystal_symmetry.unit_cell(),
       out=out)
    return lower_bounds,upper_bounds,unique_closest_sites+coordinate_offset
  
  def get_low_res_map_data(sites_cart=None,
      crystal_symmetry=None,
      d_min=None,
      out=sys.stdout):
  
      from cctbx import xray
      xrs,scatterers=set_up_xrs(crystal_symmetry=crystal_symmetry)
      unit_cell=crystal_symmetry.unit_cell()
      sites_fract=unit_cell.fractionalize(sites_cart)
      for xyz_fract in sites_fract:
        scatterers.append( xray.scatterer(scattering_type="H", label="H",
          site=xyz_fract, u=0, occupancy=1.0))
      xrs = xray.structure(xrs, scatterers=scatterers)
      # generate f_array to d_min with xrs
      f_array= xrs.structure_factors(d_min=d_min, anomalous_flag=False).f_calc()
      weight_f_array=f_array.structure_factors_from_scatterers(
        algorithm = 'direct',
        xray_structure = xrs).f_calc()
  
      return get_map_from_map_coeffs(map_coeffs=weight_f_array,
        crystal_symmetry=crystal_symmetry)
  
  def get_bounds_for_helical_symmetry(params,
       box=None,crystal_symmetry=None):
    original_cell=box.map_data.all()
    new_cell=box.map_box.all()
    z_first=box.gridding_first[2]
    z_last=box.gridding_last[2]
    assert z_last>=z_first
    z_middle=(z_first+z_last)//2
    delta_z=crystal_symmetry.unit_cell().parameters()[5]/box.map_data.all()[2]
    n_z_max= int(0.5+
     params.map_modification.restrict_z_distance_for_helical_symmetry/delta_z)
    new_z_first=max(z_first,z_middle-n_z_max)
    new_z_last=min(z_last,z_middle+n_z_max)
    lower_bounds=deepcopy(box.gridding_first)
    upper_bounds=deepcopy(box.gridding_last)
    lower_bounds[2]=new_z_first
    upper_bounds[2]=new_z_last
  
    return lower_bounds,upper_bounds
  
  def check_memory(map_data,ratio_needed,maximum_fraction_to_use=0.90,
      maximum_map_size=1,
      out=sys.stdout):
    map_size=map_data.size()/(1024*1024*1024)
    if maximum_map_size and map_size>maximum_map_size:
       raise Sorry("Maximum map size for this tool is %s GB" %(maximum_map_size))
    needed_memory=ratio_needed*map_size
    from libtbx.utils import guess_total_memory # returns total memory
  
    bytes_total_memory=guess_total_memory()
    if bytes_total_memory:
      total_memory=bytes_total_memory/(1024*1024*1024)
    else:
      total_memory=None
    print("\nMap size is " +\
        "%.2f GB.  This will require about %.1f GB of memory" %(
        map_size,needed_memory) +"\nfor this stage of analysis\n", file=out)
    if total_memory:
      print("Total memory on this computer is about %.1f GB." %(
        total_memory), file=out)
      if (needed_memory>= 0.5* total_memory):
        print("\n ** WARNING:  It is possible that this computer may not"+\
         " have **\n *** sufficient memory to complete this job. ***\n", file=out)
      if (needed_memory >= maximum_fraction_to_use*total_memory):
        raise Sorry("This computer does not have sufficient "+
          "memory (%.0f GB needed) \nto run this job" %(needed_memory))
  
  def get_params(args,map_data=None,crystal_symmetry=None,
      half_map_data_list=None,
      sharpening_target_pdb_inp=None,
      ncs_object=None,
      sequence=None,
      out=sys.stdout):
  
    params=get_params_from_args(args)
  
    print("\nSegment_and_split_map\n", file=out)
    print("Command used: %s\n" %(
     " ".join(['segment_and_split_map']+args)), file=out)
    master_params.format(python_object=params).show(out=out)
  
    # Set space-group defaults
    if params.crystal_info.use_sg_symmetry:
      if params.map_modification.restrict_map_size is None:
        params.map_modification.restrict_map_size=False
      if params.crystal_info.is_crystal is None:
        params.crystal_info.is_crystal=True
    else:
      if params.map_modification.restrict_map_size is None:
        params.map_modification.restrict_map_size=True
      if params.crystal_info.is_crystal is None:
        params.crystal_info.is_crystal=False
  
    # Turn off files if desired
    if params.control.write_files is False:
      params.output_files.magnification_map_file=None
  
      params.output_files.magnification_map_file = None
      params.output_files.magnification_ncs_file = None
      params.output_files.shifted_map_file = None
      params.output_files.shifted_sharpened_map_file = None
      params.output_files.sharpened_map_file = None
      params.output_files.shifted_pdb_file = None
      params.output_files.shifted_ncs_file = None
      params.output_files.shifted_used_ncs_file = None
      params.output_files.box_map_file = None
      params.output_files.box_mask_file = None
      params.output_files.write_output_maps = False
      params.output_files.remainder_map_file = None
      params.output_files.output_info_file = None
      params.output_files.restored_pdb = None
      params.output_files.output_weight_map_pickle_file = None
  
  
    from cctbx.maptbx.auto_sharpen import set_sharpen_params
    params=set_sharpen_params(params,out)
  
  
    if params.input_files.seq_file and not params.crystal_info.sequence and \
        not sequence:
      if not params.crystal_info.sequence:
        if sequence:
          params.crystal_info.sequence=sequence
        else:
          params.crystal_info.sequence=open(params.input_files.seq_file).read()
      print("Read sequence from %s" %(params.input_files.seq_file), file=out)
  
    if not params.crystal_info.resolution and (
       params.map_modification.b_iso is not None or \
        params.map_modification.auto_sharpen
        or params.map_modification.resolution_dependent_b or
        params.map_modification.b_sharpen):
      raise Sorry("Need resolution for segment_and_split_map with sharpening")
  
    if params.map_modification.auto_sharpen and (
        params.map_modification.b_iso is not None or
        params.map_modification.b_sharpen is not None or
        params.map_modification.resolution_dependent_b is not None):
      print("Turning off auto_sharpen as it is not compatible with "+\
          "b_iso, \nb_sharpen, or resolution_dependent_b", file=out)
      params.map_modification.auto_sharpen=False
  
    if params.control.write_files and \
         params.output_files.output_directory and  \
       (not os.path.isdir(params.output_files.output_directory)):
        os.mkdir(params.output_files.output_directory)
    if not params.output_files.output_directory:
      params.output_files.output_directory=""
  
    # Test to see if we can use adjusted_sa as target and use box_map with it
    if (params.map_modification.residual_target=='adjusted_sa' or
       params.map_modification.sharpening_target=='adjusted_sa') and \
       (params.map_modification.box_in_auto_sharpen or
         params.map_modification.density_select_in_auto_sharpen):
      print("Checking to make sure we can use adjusted_sa as target...", end=' ', file=out)
      try:
        from phenix.autosol.map_to_model import iterated_solvent_fraction
      except Exception as e:
        raise Sorry("Please either set box_in_auto_sharpen=False and "+
         "\ndensity_select_in_auto_sharpen=False or \n"+\
        "set residual_target=kurtosis and sharpening_target=kurtosis")
      print("OK", file=out)
  
    if not half_map_data_list: half_map_data_list=[]
  
    if params.input_files.info_file:
      map_data=None
      pdb_hierarchy=None
      from libtbx import easy_pickle
      print("Loading tracking data from %s" %(
        params.input_files.info_file), file=out)
      tracking_data=easy_pickle.load(params.input_files.info_file)
      return params,map_data,half_map_data_list,pdb_hierarchy,tracking_data,None
    else:
      tracking_data=info_object()
      tracking_data.set_params(params)
  
    # PDB file
    if params.input_files.pdb_file:
      print("\nInput PDB file to be used to identify region to work with: %s\n" %(
         params.input_files.pdb_file), file=out)
      pdb_inp = iotbx.pdb.input(file_name=params.input_files.pdb_file)
      pdb_hierarchy = pdb_inp.construct_hierarchy()
      pdb_atoms = pdb_hierarchy.atoms()
      pdb_atoms.reset_i_seq()
      tracking_data.set_input_pdb_info(file_name=params.input_files.pdb_file,
        n_residues=pdb_hierarchy.overall_counts().n_residues)
    else:
      pdb_hierarchy=None
  
    if map_data:
      pass # ok
    elif params.input_files.map_file:
      from iotbx import mrcfile
      ccp4_map=iotbx.mrcfile.map_reader(
      file_name=params.input_files.map_file)
      if not crystal_symmetry:
        crystal_symmetry=ccp4_map.crystal_symmetry() # 2018-07-18
        tracking_data.set_full_crystal_symmetry(
          crystal.symmetry(ccp4_map.unit_cell().parameters(),
           ccp4_map.space_group_number))
        tracking_data.set_full_unit_cell_grid(ccp4_map.unit_cell_grid)
      map_data=ccp4_map.data.as_double()
    else:
      raise Sorry("Need ccp4 map")
    if not crystal_symmetry:
      raise Sorry("Need crystal_symmetry")
  
    if params.input_files.half_map_file:
      if len(params.input_files.half_map_file) != 2:
        raise Sorry("Please supply none or two half_map_file values")
  
      from iotbx import mrcfile
      half_map_data_list=[]
      half_map_data_list.append(iotbx.mrcfile.map_reader(
         file_name=params.input_files.half_map_file[0]).data.as_double())
      half_map_data_list.append(iotbx.mrcfile.map_reader(
         file_name=params.input_files.half_map_file[1]).data.as_double())
  
    # Get the NCS object
    ncs_obj,dummy_tracking_data=get_ncs(params=params,
      ncs_object=ncs_object,out=out)
  
    if (not params.map_modification.auto_sharpen or
         params.map_modification.b_iso is not None) and (
        not params.crystal_info.molecular_mass and
        not params.crystal_info.solvent_content and
        not params.input_files.seq_file and not params.crystal_info.sequence and
        not sequence):
      params.crystal_info.solvent_content=get_iterated_solvent_fraction(
          crystal_symmetry=crystal_symmetry,
          verbose=params.control.verbose,
          resolve_size=params.control.resolve_size,
          mask_padding_fraction=\
             params.segmentation.mask_padding_fraction,
          fraction_of_max_mask_threshold=\
             params.segmentation.fraction_of_max_mask_threshold,
          cell_cutoff_for_solvent_from_mask=\
             params.segmentation.cell_cutoff_for_solvent_from_mask,
          mask_resolution=params.crystal_info.resolution,
          map=map_data,
          out=out)
      if params.crystal_info.solvent_content:
        print("Estimated solvent content: %.2f" %(
          params.crystal_info.solvent_content), file=out)
      else:
        raise Sorry("Unable to estimate solvent content...please supply "+
          "solvent_content \nor molecular_mass")
  
    if params.map_modification.auto_sharpen or \
          params.map_modification.b_iso is not None or \
          params.map_modification.b_sharpen is not None or \
          params.map_modification.resolution_dependent_b is not None:
        # Sharpen the map
        print("Auto-sharpening map before using it", file=out)
        local_params=deepcopy(params)
        if tracking_data.solvent_fraction: # XXX was previously always done but may not have been set
          local_params.crystal_info.solvent_content=tracking_data.solvent_fraction
        from cctbx.maptbx.auto_sharpen import run as auto_sharpen
        acc=map_data.accessor()
        map_data,new_map_coeffs,new_crystal_symmetry,new_si=auto_sharpen(
           args=[],params=local_params,
          map_data=map_data,
          crystal_symmetry=crystal_symmetry,
          write_output_files=False,
          pdb_inp=sharpening_target_pdb_inp,
          ncs_obj=ncs_obj,
          return_map_data_only=False,
          return_unshifted_map=True,
          half_map_data_list=half_map_data_list,
          n_residues=tracking_data.n_residues,
          ncs_copies=ncs_obj.max_operators(),
          out=out)
        tracking_data.b_sharpen=new_si.b_sharpen
        if not tracking_data.solvent_fraction:
          tracking_data.solvent_fraction=new_si.solvent_fraction
  
        if tracking_data.params.output_files.sharpened_map_file:
          sharpened_map_file=os.path.join(
              tracking_data.params.output_files.output_directory,
              tracking_data.params.output_files.sharpened_map_file)
          sharpened_map_data=map_data.deep_copy()
          if acc is not None:  # offset the map to match original if possible
            sharpened_map_data.reshape(acc)
  
          print("Gridding of sharpened map:", file=out)
          print("Origin: ",sharpened_map_data.origin(), file=out)
          print("All: ",sharpened_map_data.all(), file=out)
          print("\nWrote sharpened map in original location with "+\
               "origin at %s\nto %s" %(
             str(sharpened_map_data.origin()),sharpened_map_file), file=out)
  
          # NOTE: original unit cell and grid
          write_ccp4_map(tracking_data.full_crystal_symmetry,
            sharpened_map_file,sharpened_map_data,
            output_unit_cell_grid=tracking_data.full_unit_cell_grid,)
          params.input_files.map_file=sharpened_map_file # overwrite map_file name here
  
        # done with any sharpening
        params.map_modification.auto_sharpen=False# so we don't do it again later
        params.map_modification.b_iso=None
        params.map_modification.b_sharpen=None
        params.map_modification.resolution_dependent_b=None
        if params.control.sharpen_only:
          print("Stopping after sharpening", file=out)
          return
  
    # check on size right away
    if params.control.memory_check:
      # map_box and mask generation use about 50GB of memory for
      #    map with 1 billion elements
      check_memory(map_data=map_data,maximum_map_size=None,
        ratio_needed=50,out=out)
  
    if params.map_modification.magnification and \
         params.map_modification.magnification!=1.0:
      print("\nAdjusting magnification by %7.3f\n" %(
         params.map_modification.magnification), file=out)
  
      if ncs_obj:
        # Magnify ncs
        print("NCS before applying magnification...", file=out)
        ncs_obj.format_all_for_group_specification(out=out)
        ncs_obj=ncs_obj.adjust_magnification(
          magnification=params.map_modification.magnification)
        if params.output_files.magnification_ncs_file:
          file_name=os.path.join(params.output_files.output_directory,
            params.output_files.magnification_ncs_file)
          print("Writing NCS after magnification of %7.3f to %s" %(
            params.map_modification.magnification,file_name), file=out)
          ncs_obj.format_all_for_group_specification(out=out)
          ncs_obj.format_all_for_group_specification(file_name=file_name)
          params.input_files.ncs_file=file_name
        else:
          raise Sorry("Need magnification_ncs_file defined if magnification is"+
            " applied \nto input NCS file")
  
      # Magnify map
      shrunk_uc = []
      for i in range(3):
        shrunk_uc.append(
         crystal_symmetry.unit_cell().parameters()[i] *
            params.map_modification.magnification )
      uc_params=crystal_symmetry.unit_cell().parameters()
      from cctbx import uctbx
      new_unit_cell=uctbx.unit_cell(
        parameters=(shrunk_uc[0],shrunk_uc[1],shrunk_uc[2],
            uc_params[3],uc_params[4],uc_params[5]))
      print("Original unit cell: (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %(
        crystal_symmetry.unit_cell().parameters()), file=out)
      crystal_symmetry=crystal.symmetry(
        unit_cell=new_unit_cell,
        space_group=crystal_symmetry.space_group())
      print("New unit cell:      (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %(
        crystal_symmetry.unit_cell().parameters()), file=out)
  
      # magnify original unit cell too..
      cell=list(tracking_data.full_crystal_symmetry.unit_cell().parameters())
      for i in range(3):
        cell[i]=cell[i]*params.map_modification.magnification
      tracking_data.set_full_crystal_symmetry(
          crystal.symmetry(tuple(cell),ccp4_map.space_group_number))
      print("New original (full unit cell): "+\
          "  (%7.4f, %7.4f, %7.4f, %7.4f, %7.4f, %7.4f)" %(
        tracking_data.full_crystal_symmetry.unit_cell.parameters()), file=out)
  
      if params.output_files.magnification_map_file:
        file_name=os.path.join(params.output_files.output_directory,
          params.output_files.magnification_map_file)
        # write out magnified map (our working map) (before shifting it)
        print("\nWriting magnification map (input map with "+\
          "magnification of %7.3f \n" %(params.map_modification.magnification) +\
          "applied) to %s \n" %(file_name), file=out)
        #write_ccp4_map(crystal_symmetry,file_name,map_data)
        #  NOTE: original unit cell and grid
        write_ccp4_map(tracking_data.full_crystal_symmetry,
          file_name,map_data,
          output_unit_cell_grid=tracking_data.original_unit_cell_grid,)
        params.input_files.map_file=file_name
      else:
        raise Sorry("Need a file name to write out magnification_map_file")
      params.map_modification.magnification=None  # no longer need it.
  
    tracking_data.set_input_map_info(file_name=params.input_files.map_file,
      crystal_symmetry=crystal_symmetry,
      origin=map_data.origin(),
      all=map_data.all())
    tracking_data.set_crystal_symmetry(crystal_symmetry=crystal_symmetry)
    tracking_data.set_original_crystal_symmetry(crystal_symmetry=crystal_symmetry)
    tracking_data.set_accessor(acc=map_data.accessor())
  
    # Save center of map
    map_symmetry_center=get_center_of_map(map_data,crystal_symmetry)
  
    # Check for helical ncs...if present we may try to cut map at +/- 1 turn
    params.map_modification.restrict_z_distance_for_helical_symmetry=\
       get_max_z_range_for_helical_symmetry(params,out=out)
  
    # either use map_box with density_select=True or just shift the map
    if  params.segmentation.density_select:
      print("\nTrimming map to density...", file=out)
      args= ["output_format=ccp4"]
      if params.segmentation.density_select_threshold is not None:
        print("Threshold for density selection will be: %6.2f \n"%(
         params.segmentation.density_select_threshold), file=out)
        args.append("density_select_threshold=%s" %(
           params.segmentation.density_select_threshold))
      if params.segmentation.get_half_height_width is not None:
        args.append("get_half_height_width=%s" %(
          params.segmentation.get_half_height_width))
      if params.input_files.ncs_file:
        args.append("symmetry_file=%s" %(params.input_files.ncs_file))
      if params.input_files.pdb_file:
        args.append("pdb_file=%s" %(params.input_files.pdb_file))
      args.append("ccp4_map_file=%s" %(params.input_files.map_file))
      file_name_prefix=os.path.join(params.output_files.output_directory,
         "density_select")
      args.append("output_file_name_prefix=%s" %(file_name_prefix))
      from mmtbx.command_line.map_box import run as run_map_box
      args.append("keep_input_unit_cell_and_grid=False") # for new defaults
  
      if params.segmentation.lower_bounds and params.segmentation.upper_bounds:
        bounds_supplied=True
        print("\nRunning map_box with supplied bounds", file=out)
        box=run_map_box(args,
            map_data=map_data,
            ncs_object=ncs_obj,
            crystal_symmetry=crystal_symmetry,
            lower_bounds=params.segmentation.lower_bounds,
            upper_bounds=params.segmentation.upper_bounds,
            write_output_files=params.output_files.write_output_maps,
            log=out)
      else:
        bounds_supplied=False
        box=run_map_box(["density_select=True"]+args,
         map_data=map_data,
         crystal_symmetry=crystal_symmetry,
         ncs_object=ncs_obj,
         write_output_files=params.output_files.write_output_maps,
         log=out)
  
      # Run again to select au box
      shifted_unique_closest_sites=None
      selected_au_box=None
      if params.segmentation.select_au_box is None and  box.ncs_object and \
        box.ncs_object.max_operators() >= params.segmentation.n_ops_to_use_au_box:
        params.segmentation.select_au_box=True
        print("Setting select_au_box to True as there are %d operators" %(
          box.ncs_object.max_operators()), file=out)
      if params.segmentation.select_au_box and not bounds_supplied:
        lower_bounds,upper_bounds,unique_closest_sites=get_bounds_for_au_box(
           params, box=box,out=out) #unique_closest_sites relative to original map
        if lower_bounds and upper_bounds:
          bounds_supplied=True
          selected_au_box=True
          score,ncs_cc=score_ncs_in_map(map_data=box.map_box,
            allow_score_with_pg=False,
            sites_orth=unique_closest_sites+box.shift_cart,
            ncs_object=box.ncs_object,ncs_in_cell_only=True,
            crystal_symmetry=box.box_crystal_symmetry,out=null_out())
          print("NCS CC before rerunning box: %7.2f   SCORE: %7.1f OPS: %d " %(
           ncs_cc,score,box.ncs_object.max_operators()), file=out)
  
          print("\nRunning map-box again with boxed range ...", file=out)
          del box
          box=run_map_box(args,lower_bounds=lower_bounds,
            map_data=map_data,
            crystal_symmetry=crystal_symmetry,
            ncs_object=ncs_obj,
            upper_bounds=upper_bounds, log=out)
          box.map_box=box.map_box.as_double()  # Do we need double?
          shifted_unique_closest_sites=unique_closest_sites+box.shift_cart
  
      # Or run again for helical symmetry
      elif params.map_modification.restrict_z_distance_for_helical_symmetry and \
         not bounds_supplied:
        bounds_supplied=True
        lower_bounds,upper_bounds=get_bounds_for_helical_symmetry(params,
           crystal_symmetry=crystal_symmetry,box=box)
        print("\nRunning map-box again with restricted Z range ...", file=out)
        box=run_map_box(args,
          map_data=map_data,
          crystal_symmetry=crystal_symmetry,
          ncs_object=ncs_obj,
          lower_bounds=lower_bounds,upper_bounds=upper_bounds,
          write_output_files=params.output_files.write_output_maps,
          log=out)
  
      #-----------------------------
  
      if bounds_supplied and box.ncs_object:
        print("Selecting remaining NCS operators", file=out)
        box.ncs_object=select_remaining_ncs_ops(
          map_data=box.map_box,
          crystal_symmetry=box.box_crystal_symmetry,
          closest_sites=shifted_unique_closest_sites,
          random_points=params.reconstruction_symmetry.random_points,
          ncs_object=box.ncs_object,
          out=out)
  
        score,ncs_cc=score_ncs_in_map(map_data=box.map_box,
            allow_score_with_pg=False,
            ncs_object=box.ncs_object,ncs_in_cell_only=True,
            sites_orth=shifted_unique_closest_sites,
            crystal_symmetry=box.box_crystal_symmetry,out=null_out())
  
        if score is not None:
          print("NCS CC after selections: %7.2f   SCORE: %7.1f  OPS: %d" %(
           ncs_cc,score,box.ncs_object.max_operators()), file=out)
      #-----------------------------
  
      origin_shift=box.shift_cart
      # Note: moving cell with (0,0,0) in middle to (0,0,0) at corner means
      #   total_shift_cart and origin_shift both positive
      map_data=box.map_box
      map_data=scale_map(map_data,out=out)
      crystal_symmetry=box.box_crystal_symmetry
      print("New unit cell: %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f " %(
        crystal_symmetry.unit_cell().parameters()), file=out)
      tracking_data.set_crystal_symmetry(crystal_symmetry=crystal_symmetry)
      print("Moving origin to (0,0,0)", file=out)
      print("Adding (%8.2f,%8.2f,%8.2f) to all coordinates\n"%(
          tuple(origin_shift)), file=out)
      # NOTE: size and cell params are now different!
      tracking_data.set_box_map_bounds_first_last(
         box.gridding_first,box.gridding_last)
  
      new_half_map_data_list=[]
      ii=0
      for hm in half_map_data_list:
        ii+=1
        hm=hm.shift_origin() # shift if necessary
        hm=box.cut_and_copy_map(map_data=hm).as_double()
        hm.reshape(flex.grid(hm.all()))
        new_half_map_data_list.append(hm)
        cutout_half_map_file=os.path.join(params.output_files.output_directory,
         "cutout_half_map_%s.ccp4" %(ii))
        print("Writing cutout half_map data to %s" %(cutout_half_map_file), file=out)
        write_ccp4_map(crystal_symmetry,cutout_half_map_file,new_half_map_data_list[-1])
  
      half_map_data_list=new_half_map_data_list
      if params.map_modification.soft_mask:
        mask_data,map_data,half_map_data_list,\
        soft_mask_solvent_fraction,smoothed_mask_data,\
        original_box_map_data=\
         get_and_apply_soft_mask_to_maps(
          resolution=params.crystal_info.resolution,
          wang_radius=params.crystal_info.wang_radius,
          buffer_radius=params.crystal_info.buffer_radius,
          map_data=map_data,crystal_symmetry=crystal_symmetry,
          half_map_data_list=half_map_data_list,
          out=out)
        print("\nSolvent fraction from soft mask procedure: %7.2f (not used)\n" %(
          soft_mask_solvent_fraction), file=out)
      shifted_ncs_object=box.ncs_object
      if not shifted_ncs_object or shifted_ncs_object.max_operators()<2:
        from mmtbx.ncs.ncs import ncs
        shifted_ncs_object=ncs()
        shifted_ncs_object.set_unit_ncs()
    else:  # shift if necessary...
      shift_needed = not \
          (map_data.focus_size_1d() > 0 and map_data.nd() == 3 and
           map_data.is_0_based())
  
      a,b,c = crystal_symmetry.unit_cell().parameters()[:3]
      N_ = map_data.all()
      O_ =map_data.origin()
      sx,sy,sz= O_[0]/N_[0], O_[1]/N_[1], O_[2]/N_[2]
      # Note: If (0,0,0) is in the middle of the box, origin at sx,sy,sz
      #  is negative, shift of coordinates will be positive
      sx_cart,sy_cart,sz_cart=crystal_symmetry.unit_cell().orthogonalize(
         [sx,sy,sz])
      print("Origin for input map is at (%8.2f,%8.2f,%8.2f)" % (
        sx_cart,sy_cart,sz_cart), file=out)
      print("Cell dimensions of this map are: (%8.2f,%8.2f,%8.2f)" % (a,b,c), file=out)
      if shift_needed:
        if(not crystal_symmetry.space_group().type().number() in [0,1]):
            raise RuntimeError("Not implemented")
        origin_shift=[-sx_cart,-sy_cart,-sz_cart] # positive if (0,0,0) in middle
        print("Adding (%8.2f,%8.2f,%8.2f) to all coordinates"%(
          tuple(origin_shift))+" to put origin at (0,0,0)\n", file=out)
  
        map_data=map_data.shift_origin()
        new_half_map_data_list=[]
        for hm in half_map_data_list:
          new_half_map_data_list.append(hm.shift_origin())
        half_map_data_list=new_half_map_data_list
      else:
        origin_shift=(0.,0.,0.)
  
      # Get NCS object if any
      if params.input_files.ncs_file and not ncs_obj:
        ncs_obj,dummy_obj=get_ncs(file_name=params.input_files.ncs_file)
      if ncs_obj:
        shifted_ncs_object=ncs_obj.coordinate_offset(
          coordinate_offset=matrix.col(origin_shift)) # shift to match shifted map
      else:
        from mmtbx.ncs.ncs import ncs
        shifted_ncs_object=ncs()
        shifted_ncs_object.set_unit_ncs()
  
  
    update_tracking_data_with_sharpening(
               map_data=map_data,
               tracking_data=tracking_data,out=out)
  
    # Set origin shift now
    tracking_data.set_origin_shift(origin_shift)
  
    map_symmetry_center=matrix.col(map_symmetry_center)+matrix.col(origin_shift) # New ctr
  
    if shifted_ncs_object and params.control.check_ncs:
      ncs_obj_to_check=shifted_ncs_object
    else:
      ncs_obj_to_check=None
  
    found_ncs=False
    if params.reconstruction_symmetry.symmetry or ncs_obj_to_check or \
       params.reconstruction_symmetry.optimize_center:
      looking_for_ncs=True
      new_ncs_obj,ncs_cc,ncs_score=run_get_ncs_from_map(params=params,
        map_data=map_data,
        map_symmetry_center=map_symmetry_center,
        crystal_symmetry=crystal_symmetry,
        ncs_obj=ncs_obj_to_check,
        out=out,
        )
  
      if new_ncs_obj:
        found_ncs=True
        shifted_ncs_object=new_ncs_obj.deep_copy()
        # offset this back to where it would have been before the origin offset..
        new_ncs_obj=new_ncs_obj.coordinate_offset(
         coordinate_offset=-1*matrix.col(origin_shift))
        # XXX save it in tracking_data
  
        if params.output_files.output_directory:
          if not os.path.isdir(params.output_files.output_directory):
            os.mkdir(params.output_files.output_directory)
          file_name=os.path.join(params.output_files.output_directory,
            'ncs_from_map.ncs_spec')
          f=open(file_name,'w')
          new_ncs_obj.format_all_for_group_specification(out=f)
          f.close()
          print("Wrote NCS operators (for original map) to %s" %(file_name), file=out)
          if not params.control.check_ncs:
            params.input_files.ncs_file=file_name # set it
  
    else:
      looking_for_ncs=False
  
    if params.control.check_ncs:
      print("Done checking NCS", file=out)
      return params,map_data,half_map_data_list,pdb_hierarchy,tracking_data,None
  
    if looking_for_ncs and (not found_ncs) and \
           params.reconstruction_symmetry.symmetry.upper() not in ['ANY','ALL']:
        raise Sorry(
          "Unable to identify %s symmetry automatically in this map." %(
          params.reconstruction_symmetry.symmetry)+
          "\nPlease supply a symmetry file with symmetry matrices.")
  
    if params.segmentation.expand_size is None:
      params.segmentation.expand_size=estimate_expand_size(
         crystal_symmetry=crystal_symmetry,
         map_data=map_data,
         expand_target=params.segmentation.expand_target,
         out=out)
  
    if params.output_files.output_info_file and params.control.shift_only:
      write_info_file(params=params,tracking_data=tracking_data,out=out)
  
    return params,map_data,half_map_data_list,pdb_hierarchy,\
       tracking_data,shifted_ncs_object
  
  def write_info_file(params=None,tracking_data=None,out=sys.stdout):
      # write out the info file
      from libtbx import easy_pickle
      tracking_data.show_summary(out=out)
      print("\nWriting summary information to: %s" %(
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.output_info_file)), file=out)
      print("\nTo restore original position of a PDB file built into these maps, use:", file=out)
      print("phenix.segment_and_split_map info_file=%s" %(
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.output_info_file))+" pdb_to_restore=mypdb.pdb\n", file=out)
      easy_pickle.dump(os.path.join(tracking_data.params.output_files.output_directory,params.output_files.output_info_file),
         tracking_data)
  
  
  def get_and_apply_soft_mask_to_maps(
      resolution=None,  #params.crystal_info.resolution
      wang_radius=None, #params.crystal_info.wang_radius
      buffer_radius=None, #params.crystal_info.buffer_radius
      map_data=None,crystal_symmetry=None,
      solvent_content=None,
      solvent_content_iterations=None,
      rad_smooth=None,
      half_map_data_list=None,
      out=sys.stdout):
    smoothed_mask_data=None
    if not resolution:
      raise Sorry("Need resolution for soft_mask")
  
    if not rad_smooth:
      rad_smooth=resolution
  
    print("\nApplying soft mask with smoothing radius of %s\n" %(
      rad_smooth), file=out)
    if wang_radius:
      wang_radius=wang_radius
    else:
      wang_radius=1.5*resolution
  
    if buffer_radius:
      buffer_radius=buffer_radius
    else:
      buffer_radius=2.*resolution
    original_map_data=map_data.deep_copy()
    # Check to make sure this is possible
    cell_dims=crystal_symmetry.unit_cell().parameters()[:3]
    min_cell_dim=min(cell_dims)
    if wang_radius > 0.25 * min_cell_dim or buffer_radius > 0.25 * min_cell_dim:
      raise Sorry("Cell is too small to get solvent fraction")
    mask_data,solvent_fraction=get_mask_around_molecule(map_data=map_data,
      crystal_symmetry=crystal_symmetry,
      wang_radius=wang_radius,
      solvent_content=solvent_content,
      solvent_content_iterations=solvent_content_iterations,
      buffer_radius=buffer_radius,
      return_masked_fraction=True,
      out=out)
    if mask_data:
      map_data,smoothed_mask_data=apply_soft_mask(map_data=map_data,
        mask_data=mask_data.as_double(),
        rad_smooth=rad_smooth,
        crystal_symmetry=crystal_symmetry,
        out=out)
  
      new_half_map_data_list=[]
      if not half_map_data_list: half_map_data_list=[]
      for half_map in half_map_data_list:
        half_map,smoothed_mask_data=apply_soft_mask(map_data=half_map,
          mask_data=mask_data.as_double(),
          rad_smooth=rad_smooth,
          crystal_symmetry=crystal_symmetry,
          out=out)
        new_half_map_data_list.append(half_map)
      half_map_data_list=new_half_map_data_list
    else:
      print("Unable to get mask...skipping", file=out)
    return mask_data,map_data,half_map_data_list,\
      solvent_fraction,smoothed_mask_data,original_map_data
  
  def get_ncs(params=None,tracking_data=None,file_name=None,
       ncs_object=None,out=sys.stdout):
    if not file_name:
      file_name=params.input_files.ncs_file
    if (not ncs_object or ncs_object.max_operators()<2) and file_name: print("Reading ncs from %s" %(file_name), file=out)
    is_helical_symmetry=None
    if (not ncs_object or ncs_object.max_operators()<2) and not file_name: # No ncs supplied...use just 1 ncs copy..
      from mmtbx.ncs.ncs import ncs
      ncs_object=ncs()
      ncs_object.set_unit_ncs()
      #ncs_object.display_all(log=out)
    elif (not ncs_object or ncs_object.max_operators()<2) and \
        not os.path.isfile(file_name):
      raise Sorry("The ncs file %s is missing" %(file_name))
    else: # get the ncs
      if not ncs_object:
        from mmtbx.ncs.ncs import ncs
        ncs_object=ncs()
        try: # see if we can read biomtr records
          pdb_inp=iotbx.pdb.input(file_name=file_name)
          ncs_object.ncs_from_pdb_input_BIOMT(pdb_inp=pdb_inp,log=out)
        except Exception as e: # try as regular ncs object
          ncs_object.read_ncs(file_name=file_name,log=out)
        #ncs_object.display_all(log=out)
      ncs_object.select_first_ncs_group()
      if ncs_object.max_operators()<1:
        from mmtbx.ncs.ncs import ncs
        ncs_object=ncs()
        ncs_object.set_unit_ncs()
      print("\nTotal of %d NCS operators read\n" %(
        ncs_object.max_operators()), file=out)
      if not tracking_data or not params:
        return ncs_object,None
      if ncs_object.max_operators()<2:
         print("No NCS present", file=out)
      elif ncs_object.is_helical_along_z(
         abs_tol_t=tracking_data.params.reconstruction_symmetry.abs_tol_t,
         rel_tol_t=tracking_data.params.reconstruction_symmetry.rel_tol_t,
         tol_r=tracking_data.params.reconstruction_symmetry.tol_r):
        print("This NCS is helical symmetry", file=out)
        is_helical_symmetry=True
      elif ncs_object.is_point_group_symmetry(
         abs_tol_t=tracking_data.params.reconstruction_symmetry.abs_tol_t,
         rel_tol_t=tracking_data.params.reconstruction_symmetry.rel_tol_t,
         tol_r=tracking_data.params.reconstruction_symmetry.tol_r):
        print("This NCS is point-group symmetry", file=out)
      elif params.crystal_info.is_crystal:
        print("This NCS is crystal symmetry", file=out)
      elif not (
        params.reconstruction_symmetry.require_helical_or_point_group_symmetry):
        print("WARNING: NCS is not crystal symmetry nor point-group "+\
           "symmetry nor helical symmetry", file=out)
      else:
        raise Sorry("Need point-group or helical symmetry.")
    if not ncs_object or ncs_object.max_operators()<1:
      raise Sorry("Need ncs information from an ncs_info file")
    if tracking_data:
      tracking_data.set_input_ncs_info(file_name=file_name,  # XXX may be updated ops
        number_of_operators=ncs_object.max_operators())
  
    if tracking_data and is_helical_symmetry: # update shifted_ncs_info
      if tracking_data.shifted_ncs_info: # XXX may not be needed
         shifted=True
      else:
         shifted=False
      print("Updating NCS info (shifted=%s)" %(shifted), file=out)
      tracking_data.update_ncs_info(is_helical_symmetry=True,shifted=shifted)
  
      if tracking_data.input_map_info and tracking_data.input_map_info.all:
        z_range=tracking_data.crystal_symmetry.unit_cell(). \
           parameters()[2]
        print("Extending NCS operators to entire cell (z_range=%.1f)" %(
           z_range), file=out)
        max_operators= \
            tracking_data.params.reconstruction_symmetry.max_helical_operators
        if max_operators:
          print("Maximum new number of NCS operators will be %s" %(
           max_operators), file=out)
        ncs_object.extend_helix_operators(z_range=z_range,
          max_operators=max_operators)
        #ncs_object.display_all()
        print("New number of NCS operators is: %s " %(
          ncs_object.max_operators()), file=out)
        tracking_data.update_ncs_info(
          number_of_operators=ncs_object.max_operators(),is_helical_symmetry=True,
          shifted=shifted)
    return ncs_object,tracking_data
  
  def score_threshold(b_vs_region=None,threshold=None,
       sorted_by_volume=None,n_residues=None,
       ncs_copies=None,
       fraction_occupied=None,
       solvent_fraction=None,
       map_data=None,
       residues_per_region=50,
       min_volume=None,
       min_ratio=None,
       max_ratio_to_target=None,
       min_ratio_to_target=None,
       weight_score_grid_points=1.,
       weight_score_ratio=1.0,
       weight_near_one=0.1,
       min_ratio_of_ncs_copy_to_first=None,
       target_in_all_regions=None,
       crystal_symmetry=None,
       chain_type=None,
       out=sys.stdout):
     # We want about 1 region per 50-100 residues for the biggest region.
     # One possibility is to try to maximize the median size of the N top
     # regions, where N=number of expected regions= n_residues/residues_per_region
  
     # Also note we have an idea how big a region should be (how many
     # grid points) if we make an assumption about the fractional volume that
     # should be inside a region compared to the total volume of protein/nucleic
     # acid in the region...this gives us target_in_top_regions points.
     # So using this, make the median size as close to target_in_top_regions as
     # we can.
  
     # If we have solvent fraction but not ncs_copies or n_residues, guess the
     #  number of residues and ncs copies from the volume
     if ncs_copies is not None and n_residues is not None:
       expected_regions=max(ncs_copies,
        max(1,int(0.5+n_residues/residues_per_region)))
     else:
        if chain_type in [None,'None']: chain_type="PROTEIN"
        assert crystal_symmetry is not None
        assert solvent_fraction is not None
        volume_per_residue,nres,chain_type=get_volume_of_seq(
            "A",chain_type=chain_type,out=out)
  
        expected_regions=max(1,int(0.5+(1-solvent_fraction)*\
          crystal_symmetry.unit_cell().volume()/volume_per_residue ))
        # NOTE: This is expected residues. expected_regions should be this
        #  divided by residues_per_region
        expected_regions=max(1,int(0.5+expected_regions/residues_per_region))
        ncs_copies=1
  
     target_in_top_regions=target_in_all_regions/expected_regions
  
     nn=len(sorted_by_volume)-1 # first one is total
     ok=True
  
     too_low=None  # marker for way too low
     too_high=None
  
     if nn < ncs_copies:
       ok=False #return  # not enough
  
     v1,i1=sorted_by_volume[1]
     if v1 < min_volume:
       ok=False #return
  
     if v1 > max_ratio_to_target*target_in_top_regions:
       ok=False #return
       too_low=True
  
     if v1 < min_volume or v1 < 0.1*min_ratio_to_target*target_in_top_regions:
       # way too high
       too_high=True
  
  
     # there should be about ncs_copies copies of each size region if ncs_copies>1
     if ncs_copies>1:
       v2,i2=sorted_by_volume[max(1,min(ncs_copies,nn))]
       score_ratio=v2/v1  # want it to be about 1
       if score_ratio < min_ratio_of_ncs_copy_to_first:
         ok=False #return  # not allowed
     else:
       score_ratio=1.0 # for ncs_copies=1
  
     nn2=min(nn,max(1,(expected_regions+1)//2))
     median_number,iavg=sorted_by_volume[nn2]
  
     # number in each region should be about target_in_top_regions
  
     if median_number > target_in_top_regions:
       score_grid_points=target_in_top_regions/max(1.,median_number)
     else:
       score_grid_points=median_number/target_in_top_regions
  
     if v1> target_in_top_regions:
       score_grid_points_b=target_in_top_regions/max(1.,v1)
     else:
       score_grid_points_b=v1/target_in_top_regions
  
     score_grid_points=0.5*(score_grid_points+score_grid_points_b)
  
     score_grid_points=score_grid_points**2  # maybe even **3
  
     if threshold>1.:
       score_near_one=1./threshold
     else:
       score_near_one=threshold
  
     # Normalize weight_score_ratio by target_in_top_regions:
     sc=min(1.,0.5*median_number/max(1,target_in_top_regions))
     overall_score=(
       (sc*weight_score_ratio*score_ratio+
       weight_score_grid_points*score_grid_points+
       weight_near_one*score_near_one
         ) /
       (weight_score_ratio+weight_score_grid_points+weight_near_one))
  
     half_expected_regions=max(1,(1+expected_regions)//2)
     ratio=sorted_by_volume[min(len(sorted_by_volume)-1,half_expected_regions)][0]/v1
  
     if ok and v1 >= target_in_top_regions/2 and \
          len(sorted_by_volume)>half_expected_regions:
       last_volume=sorted_by_volume[half_expected_regions][0]
       if ratio >=min_ratio and \
           last_volume>=min_volume:
         has_sufficient_regions=True
       else:
         has_sufficient_regions=False
     else:
         has_sufficient_regions=False
  
  
     print("%7.2f  %5.2f   %5d     %4d    %5d     %5d     %6.3f   %5s    %5.3f  %s  %s" %(
         b_vs_region.b_iso,threshold,target_in_top_regions,expected_regions,
         v1,median_number,ratio,has_sufficient_regions,overall_score,ok,nn), file=out)
  
     if not b_vs_region.b_iso in b_vs_region.b_vs_region_dict.keys():
       b_vs_region.b_vs_region_dict[b_vs_region.b_iso]={}
       b_vs_region.sa_sum_v_vs_region_dict[b_vs_region.b_iso]={}
       b_vs_region.sa_nn_vs_region_dict[b_vs_region.b_iso]={}
       b_vs_region.sa_ratio_b_vs_region_dict[b_vs_region.b_iso]={}
     b_vs_region.b_vs_region_dict[b_vs_region.b_iso][threshold]=nn
     b_vs_region.sa_nn_vs_region_dict[b_vs_region.b_iso][threshold]=None
     b_vs_region.sa_ratio_b_vs_region_dict[b_vs_region.b_iso][threshold]=None
  
     return overall_score,has_sufficient_regions,\
        too_low,too_high,expected_regions,ok
  
  
  def choose_threshold(b_vs_region=None,map_data=None,
       fraction_occupied=None,
       solvent_fraction=None,
       n_residues=None,
       ncs_copies=None,
       scale=0.95,
       calculate_sa=None, # calculate surface area of top sa_percent of target
       sa_percent=None, # calculate surface area of top sa_fraction of target
       density_threshold=None,
       starting_density_threshold=None,
       wrapping=None,
       residues_per_region=None,
       min_volume=None,
       min_ratio=None,
       max_ratio_to_target=None,
       min_ratio_to_target=None,
       min_ratio_of_ncs_copy_to_first=None,
       verbose=None,
       crystal_symmetry=None,
       chain_type=None,
       out=sys.stdout):
  
    best_threshold=None
    best_threshold_has_sufficient_regions=None
    best_score=None
    best_ok=None
  
    if not ncs_copies: ncs_copies=1
  
    print("\nChecking possible cutoffs for region identification", file=out)
    print("Scale: %7.3f" %(scale), file=out)
    used_ranges=[]
  
    # Assume any threshold that is lower than a threshold that gave a non-zero value
    #  and is zero is an upper bound on the best value.  Same the other way around
    upper_bound=1000
    lower_bound=0.0001
    best_nn=None
  
    if density_threshold is not None: # use it
       print("\nUsing input threshold of %5.2f " %(
        density_threshold), file=out)
       n_range_low_high_list=[[0,0]] # use as is
    else:
      n_range_low_high_list=[[-16,4],[-32,16],[-64,80]]
      if starting_density_threshold is not None:
        starting_density_threshold=starting_density_threshold
        print("Starting density threshold is: %7.3f" %(
           starting_density_threshold), file=out)
      else:
        starting_density_threshold=1.0
    if verbose:
      local_out=out
    else:
      from libtbx.utils import null_out
      local_out=null_out()
  
    target_in_all_regions=map_data.size()*fraction_occupied*(1-solvent_fraction)
    print("\nTarget number of points in all regions: %.0f" %(
      target_in_all_regions), file=local_out)
  
  
    local_threshold=find_threshold_in_map(target_points=int(
         target_in_all_regions),map_data=map_data)
    print("Cutoff will be threshold of %7.2f marking %7.1f%% of cell" %(
              local_threshold,100.*(1.-solvent_fraction)), file=out)
  
    print("B-iso  Threshold  Target    N     Biggest   Median     Ratio   Enough  Score   OK  Regions", file=local_out)
    unique_expected_regions=None
    for n_range_low,n_range_high in n_range_low_high_list:
      last_score=None
      for nn in range(n_range_low,n_range_high+1):
        if nn in used_ranges: continue
        used_ranges.append(nn)
        if density_threshold is not None:
          threshold=density_threshold
        else:
          threshold=starting_density_threshold*(scale**nn)
        if threshold < lower_bound or threshold > upper_bound:
          continue
  
        co,sorted_by_volume,min_b,max_b=get_co(
          map_data=map_data.deep_copy(),
           threshold=threshold,wrapping=wrapping)
  
        if len(sorted_by_volume)<2:
          score,has_sufficient_regions,too_low,too_high,expected_regions,ok=\
            None,None,None,None,None,None
          continue # don't go on
        else:
          score,has_sufficient_regions,too_low,too_high,expected_regions,ok=\
             score_threshold(b_vs_region=b_vs_region,
           threshold=threshold,
           sorted_by_volume=sorted_by_volume,
           fraction_occupied=fraction_occupied,
           solvent_fraction=solvent_fraction,
           residues_per_region=residues_per_region,
           min_volume=min_volume,
           min_ratio=min_ratio,
           max_ratio_to_target=max_ratio_to_target,
           min_ratio_to_target=min_ratio_to_target,
           min_ratio_of_ncs_copy_to_first=min_ratio_of_ncs_copy_to_first,
           ncs_copies=ncs_copies,
           n_residues=n_residues,
           map_data=map_data,
           target_in_all_regions=target_in_all_regions,
           crystal_symmetry=crystal_symmetry,
           chain_type=chain_type,
           out=local_out)
        if expected_regions:
          unique_expected_regions=max(1,
           (ncs_copies-1+expected_regions)//ncs_copies)
        if too_high and threshold<upper_bound:
          upper_bound=threshold
        if too_low and threshold>lower_bound:
          lower_bound=threshold
        if score is None:
          if best_threshold and best_threshold_has_sufficient_regions:
            if threshold >best_threshold: # new upper bound
             upper_bound=threshold
            elif threshold <best_threshold: # new lower bound
             lower_bound=threshold
        elif  (ok or not best_ok) and  \
              (best_score is None or score > best_score):
          best_threshold=threshold
          best_threshold_has_sufficient_regions=has_sufficient_regions
          best_score=score
          best_ok=ok
  
    if best_threshold is not None:
      print("\nBest threshold: %5.2f\n" %(best_threshold), file=out)
      return best_threshold,unique_expected_regions,best_score,best_ok
    elif density_threshold is not None: # use it anyhow
      return density_threshold,unique_expected_regions,None,None
    else:
      return None,unique_expected_regions,None,None
  
  def get_co(map_data=None,threshold=None,wrapping=None):
    co=maptbx.connectivity(map_data=map_data,threshold=threshold,
           wrapping=wrapping)
    regions=co.regions()
    rr=list(range(0,co.regions().size()))
  
    regions_0=regions[0]
    rr_0=rr[0]
    regions=regions[1:]
    rr=rr[1:]
    if rr:
      z = zip(regions,rr)
      sorted_by_volume = sorted(z, key=lambda x: x[0], reverse=True)
    else:
      sorted_by_volume = []
    sorted_by_volume=[(regions_0,rr_0)]+sorted_by_volume
  
    min_b, max_b = co.get_blobs_boundaries_tuples() # As grid points, not A
    return co,sorted_by_volume,min_b,max_b
  
  def get_connectivity(b_vs_region=None,
       map_data=None,
       solvent_fraction=None,
       n_residues=None,
       ncs_copies=None,
       fraction_occupied=None,
       iterate_with_remainder=None,
       min_volume=None,
       min_ratio=None,
       wrapping=None,
       residues_per_region=None,
       max_ratio_to_target=None,
       min_ratio_to_target=None,
       min_ratio_of_ncs_copy_to_first=None,
       starting_density_threshold=None,
       density_threshold=None,
       crystal_symmetry=None,
       chain_type=None,
       verbose=None,
       out=sys.stdout):
    print("\nGetting connectivity", file=out)
    libtbx.call_back(message='segment',data=None)
  
  
    # Normalize map data now to SD of the part that is not solvent
    map_data=renormalize_map_data(
       map_data=map_data,solvent_fraction=solvent_fraction)
  
    # Try connectivity at various thresholds
    # Choose one that has about the right number of grid points in top regions
    scale=0.95
    best_threshold=None
    best_scale=scale
    best_score=None
    best_ok=None
    best_unique_expected_regions=None
    for ii in range(3):
      threshold,unique_expected_regions,score,ok=choose_threshold(
       density_threshold=density_threshold,
       starting_density_threshold=starting_density_threshold,
       b_vs_region=b_vs_region,
       map_data=map_data,
       n_residues=n_residues,
       ncs_copies=ncs_copies,
       fraction_occupied=fraction_occupied,
       solvent_fraction=solvent_fraction,
       scale=scale,
       wrapping=wrapping,
       residues_per_region=residues_per_region,
       min_volume=min_volume,
       min_ratio=min_ratio,
       max_ratio_to_target=max_ratio_to_target,
       min_ratio_to_target=min_ratio_to_target,
       min_ratio_of_ncs_copy_to_first=min_ratio_of_ncs_copy_to_first,
       crystal_symmetry=crystal_symmetry,
       chain_type=chain_type,
       verbose=verbose,
       out=out)
      # Take it if it improves (score, ok)
      if threshold is not None:
       if best_score is None or  \
        ((ok or not best_ok) and (score > best_score)):
        best_score=score
        best_unique_expected_regions=unique_expected_regions
        best_ok=ok
        best_threshold=threshold
        best_scale=scale
      if best_ok or density_threshold is not None:
        break
      else:
        scale=scale**0.333 # keep trying
  
    if best_threshold is None or (
        density_threshold is not None and best_score is None):
      if iterate_with_remainder: # on first try failed
        raise Sorry("No threshold found...try with density_threshold=xxx")
      else: # on iteration...ok
        print("Note: No threshold found", file=out)
        return None,None,None,None,None,None,None,None
    else:
      starting_density_threshold=best_threshold
      # try it next time
  
    co,sorted_by_volume,min_b,max_b=get_co(
      map_data=map_data,threshold=best_threshold,wrapping=wrapping)
  
    return co,sorted_by_volume,min_b,max_b,best_unique_expected_regions,\
        best_score,threshold,starting_density_threshold
  
  def get_volume_of_seq(text,chain_type=None,out=sys.stdout):
    from iotbx.bioinformatics import chain_type_and_residues
    # get chain type and residues (or use given chain type and count residues)
    chain_type,n_residues=chain_type_and_residues(text=text,chain_type=chain_type)
    if chain_type is None and n_residues is None:
      return None,None,None
    if chain_type=='PROTEIN':
      mw_residue=110.0  # from $CDOC/matthews.doc
      density_factor=1.23   # 1.66/DENSITY-OF-PROTEIN=1.66/1.35
    else:
      mw_residue=330.0  # guess for DNA/RNA
      density_factor=1.15   # 1.66/DENSITY-OF-DNA=1.66/1.45
    return len(text)*density_factor*mw_residue,len(text),chain_type
  
  def create_rna_dna(cns_dna_rna_residue_names):
    dd={}
    for key in cns_dna_rna_residue_names.keys():
      dd[cns_dna_rna_residue_names[key]]=key
    return dd
  
  def get_solvent_content_from_seq_file(params,
      sequence=None,
      seq_file=None,
      ncs_copies=None,
      map_volume=None,
      out=sys.stdout):
    if not sequence and not os.path.isfile(seq_file):
      raise Sorry(
       "The sequence file '%s' is missing." %(seq_file))
    if not sequence:
      print("\nReading sequence from %s " %(seq_file), file=out)
      sequence=open(seq_file).read()
    from iotbx.bioinformatics import get_sequences
    sequences=get_sequences(text=sequence)
    # get unique part of these sequences
  
    from mmtbx.validation.chain_comparison import \
         extract_unique_part_of_sequences as eups
    print("Unique part of sequences:", file=out)
    copies_in_unique,base_copies,unique_sequence_dict=eups(sequences)
    all_unique_sequence=[]
    for seq in copies_in_unique.keys():
      print("Copies: %s  base copies: %s  Sequence: %s" %(
         copies_in_unique[seq],base_copies,seq))
      all_unique_sequence.append(seq)
    if base_copies != ncs_copies:
      print("NOTE: %s copies of unique portion but ncs_copies=%s" %(
         base_copies,ncs_copies), file=out)
      if ncs_copies==1:
        ncs_copies=base_copies
        print("Using ncs_copies=%s instead" %(ncs_copies), file=out)
      else:
        print("Still using ncs_copies=%s" %(ncs_copies), file=out)
  
    volume_of_chains=0.
    n_residues=0
    chain_types_considered=[]
    for seq in all_unique_sequence:
      volume,nres,chain_type=get_volume_of_seq(seq,
        chain_type=params.crystal_info.chain_type,out=out)
      if volume is None: continue
      volume_of_chains+=volume
      n_residues+=nres
      if not chain_type in chain_types_considered:
        chain_types_considered.append(chain_type)
    chain_types_considered.sort()
    print("\nChain types considered: %s\n" %(
        " ".join(chain_types_considered)), file=out)
    volume_of_molecules=volume_of_chains*ncs_copies
    n_residues_times_ncs=n_residues*ncs_copies
    solvent_fraction=1.-(volume_of_molecules/map_volume)
    solvent_fraction=max(0.001,min(0.999,solvent_fraction))
    if solvent_fraction==0.001 or solvent_fraction==0.999:
      print("NOTE: solvent fraction of %7.2f very unlikely..." %(
          solvent_fraction) + "please check ncs_copies and sequence ", file=out)
    print("Solvent content from composition: %7.2f" %(solvent_fraction), file=out)
    print("Cell volume: %.1f  NCS copies: %d   Volume of unique chains: %.1f" %(
       map_volume,ncs_copies,volume_of_chains), file=out)
    print("Total residues: %d  Volume of all chains: %.1f  Solvent fraction: %.3f "%(
         n_residues_times_ncs,volume_of_molecules,solvent_fraction), file=out)
    return solvent_fraction,n_residues,n_residues_times_ncs
  
  def get_solvent_fraction(params,
       ncs_object=None,ncs_copies=None,
       crystal_symmetry=None,tracking_data=None,out=sys.stdout):
    if tracking_data and not crystal_symmetry:
      #crystal_symmetry=tracking_data.original_crystal_symmetry not used
      crystal_symmetry=tracking_data.crystal_symmetry
    map_volume=crystal_symmetry.unit_cell().volume()
    if tracking_data and not ncs_copies:
      #ncs_copies=tracking_data.input_ncs_info.original_number_of_operators
      ncs_copies=tracking_data.input_ncs_info.number_of_operators # 2018-01-29 put back
    if not ncs_copies: ncs_copies=1
  
  
  
    if params.input_files.seq_file or params.crystal_info.sequence:
      solvent_content,n_residues,n_residues_times_ncs=\
           get_solvent_content_from_seq_file(
       params,
       sequence=params.crystal_info.sequence,
       seq_file=params.input_files.seq_file,
       ncs_copies=ncs_copies,
       map_volume=map_volume,
       out=out)
      if not params.crystal_info.solvent_content:
        params.crystal_info.solvent_content=solvent_content
        print("Solvent fraction from composition: %7.2f "%(
         params.crystal_info.solvent_content), file=out)
      else:
        print("Solvent content from parameters: %7.2f" %(
          params.crystal_info.solvent_content), file=out)
  
    else:
      if params.crystal_info.solvent_content:
        print("Solvent content from parameters: %7.2f" %(
          params.crystal_info.solvent_content), file=out)
      elif params.crystal_info.molecular_mass:
         params.crystal_info.solvent_content=\
           get_solvent_fraction_from_molecular_mass(
          crystal_symmetry=crystal_symmetry,
          molecular_mass=params.crystal_info.molecular_mass,
          out=out)
  
      else:
        print("Getting solvent content automatically.", file=out)
  
    if tracking_data:
      if params.input_files.seq_file or params.crystal_info.sequence:
        tracking_data.set_input_seq_info(file_name=params.input_files.seq_file,
         sequence=params.crystal_info.sequence,
          n_residues=n_residues)
        tracking_data.set_n_residues(
          n_residues=n_residues_times_ncs)
      if params.crystal_info.solvent_content:
        tracking_data.set_solvent_fraction(params.crystal_info.solvent_content)
  
      return tracking_data
    else:
      return params.crystal_info.solvent_content
  
  def top_key(dd):
    if not dd:
      return None,None
    elif len(dd)==1:
      return list(dd.items())[0]
    else:
      best_key=None
      best_n=None
      for key in dd.keys():
        if not best_n or dd[key] > best_n:
          best_n=dd[key]
          best_key=key
      return best_key,best_n
  
  def choose_max_regions_to_consider(params,
      sorted_by_volume=None,
      ncs_copies=None):
  
    max_per_au=params.segmentation.max_per_au
    min_ratio=params.segmentation.min_ratio
    min_volume=params.segmentation.min_volume
    # sort and eliminate regions with few points and those at end of list
    if len(sorted_by_volume)<2:
      return 0
    max_grid_points=sorted_by_volume[1][0]
    cntr=0
    for p in sorted_by_volume[1:]:
      cntr+=1
      if max_per_au and (cntr>max_per_au*ncs_copies):
        cntr-=1
        break
      v,i=p  # v=volume in grid points, i=id
      if v/max_grid_points<min_ratio or v < min_volume:
        cntr-=1
        break
    return cntr
  
  def get_edited_mask(sorted_by_volume=None,
      max_regions_to_consider=None,
      co=None,
      out=sys.stdout):
    conn_obj=co.result()
    origin=list(conn_obj.accessor().origin())
    all=list(conn_obj.accessor().all())
    conn_obj.accessor().show_summary(out)
    edited_mask=conn_obj.deep_copy()
    first=True
    edited_volume_list=[]
    original_id_from_id={}
    for i in range(1,max_regions_to_consider+1):
      v,id=sorted_by_volume[i]
      original_id_from_id[i]=id
      edited_volume_list.append(v)
      s = (conn_obj==id)
      if first:
        edited_mask=edited_mask.set_selected(~s,0)
        first=False
      edited_mask=edited_mask.set_selected(s,i)   # edited mask has ID of
           # regions, labeled in decreasing size from 1 to max_regions_to_consider
    return edited_mask,edited_volume_list,original_id_from_id
  
  def choose_subset(a,target_number=1):
    new_array=flex.vec3_double()
    assert type(new_array)==type(a)
    n=a.size()
    nskip=max(1,n//target_number)
    i=0
    for x in a:
      if i%nskip==0 or i==n-1:
       new_array.append(x)
      i+=1
    return new_array
  
  def run_get_duplicates_and_ncs(
     ncs_obj=None,
     min_b=None,
     max_b=None,
     edited_mask=None,
     original_id_from_id=None,
     edited_volume_list=None,
     max_regions_to_consider=None,
     regions_left=None,
     tracking_data=None,
     out=sys.stdout,
     ):
  
    duplicate_dict,equiv_dict,equiv_dict_ncs_copy_dict,region_range_dict,\
       region_centroid_dict,region_scattered_points_dict=\
        get_duplicates_and_ncs(
          ncs_obj=ncs_obj,
          min_b=min_b,
          max_b=max_b,
          edited_mask=edited_mask,
          edited_volume_list=edited_volume_list,
          original_id_from_id=original_id_from_id,
          max_regions_to_consider=max_regions_to_consider,
          tracking_data=tracking_data,
          out=out)
  
    # check that we have region_centroid for all values
    complete=True
    missing=[]
    for i in range(1,max_regions_to_consider+1):
      if not i in region_centroid_dict.keys():
        if (regions_left is None) or (i in regions_left):
           complete=False
           missing.append(i)
    if complete:
         return duplicate_dict,equiv_dict,equiv_dict_ncs_copy_dict,\
          region_range_dict,region_centroid_dict,\
          region_scattered_points_dict
    else:
      raise Sorry("Cannot find region-centroid for all regions? Missing: %s" %(
        missing))
  
  def copy_dict_info(from_dict,to_dict):
    for key in from_dict.keys():
      to_dict[key]=from_dict[key]
  
  def get_centroid_from_blobs(min_b=None,max_b=None,
      id=None,original_id_from_id=None):
    orig_id=original_id_from_id[id]
    upper=max_b[orig_id]
    lower=min_b[orig_id]
    avg=[]
    for u,l in zip(upper,lower):
      avg.append(0.5*(u+l))
    return avg
  
  def get_duplicates_and_ncs(
     ncs_obj=None,
     min_b=None,
     max_b=None,
     edited_mask=None,
     original_id_from_id=None,
     edited_volume_list=None,
     max_regions_to_consider=None,
     target_points_per_region=30,
     minimum_points_per_region=10,
     maximum_points_per_region=100,
     tracking_data=None,
     out=sys.stdout,
     ):
  
    unit_cell=tracking_data.crystal_symmetry.unit_cell()
    region_scattered_points_dict=get_region_scattered_points_dict(
       edited_volume_list=edited_volume_list,
       edited_mask=edited_mask,
       unit_cell=unit_cell,
       target_points_per_region=target_points_per_region,
       minimum_points_per_region=minimum_points_per_region,
       maximum_points_per_region=maximum_points_per_region)
  
  
    # Now just use the scattered points to get everything else:
    region_n_dict={}  # count of points used by region (differs from volume due
       # to the sampling)
    region_range_dict={} # keyed by region in edited_mask; range for x, y, z
    region_centroid_dict={} # keyed by region in edited_mask; range for x, y, z
    for id in region_scattered_points_dict.keys():
      sites=region_scattered_points_dict[id]
      region_n_dict[id]=sites.size()
      if region_n_dict[id]:
        region_centroid_dict[id]=list(sites.mean())
      else: # No points...use bounds from object
        region_centroid_dict[id]=get_centroid_from_blobs(min_b=min_b,
          max_b=max_b,
          id=id,original_id_from_id=original_id_from_id)
  
    # Now get NCS relationships
  
    ncs_group=ncs_obj.ncs_groups()[0]
    duplicate_dict={}  # keyed by id, number of duplicates for that region
    equiv_dict={}  # equiv_dict[id][other_id]=number_of points other_id matches
                   #  id through an ncs relationship
    equiv_dict_ncs_copy_dict={}
    for id in region_scattered_points_dict.keys():
      duplicate_dict[id]=0
      equiv_dict[id]={}
      equiv_dict_ncs_copy_dict[id]={}
  
    # Figure out which ncs operator is the identity
    identity_op=ncs_group.identity_op_id()
    print("Identity operator is %s" %(identity_op), file=out)
  
    # 2017-12-16 Score poorly if it involves a cell translation unless it
    #  is a crystal
  
    if len(ncs_group.translations_orth())>1:
      # Skip if no ncs...
      for id in region_scattered_points_dict.keys():
        for xyz_cart in region_scattered_points_dict[id]:
          n=0
          for i0 in range(len(ncs_group.translations_orth())):
            if i0==identity_op: continue
            r=ncs_group.rota_matrices_inv()[i0] # inverse maps pos 0 on to pos i
            t=ncs_group.translations_orth_inv()[i0]
  
            n+=1
            new_xyz_cart=r * matrix.col(xyz_cart) + t
            new_xyz_frac=unit_cell.fractionalize(new_xyz_cart)
            if tracking_data.params.crystal_info.use_sg_symmetry or \
              (new_xyz_frac[0]>=0 and new_xyz_frac[0]<=1 and \
               new_xyz_frac[1]>=0 and new_xyz_frac[1]<=1 and \
               new_xyz_frac[2]>=0 and new_xyz_frac[2]<=1):
              value=edited_mask.value_at_closest_grid_point(new_xyz_frac)
            else:
              value=0  # value for nothing there 2017-12-16
            if value==id:
              duplicate_dict[id]+=1
              break # only count once
            elif value>0:  # notice which one is matched
              if not value in equiv_dict[id]:
                equiv_dict[id][value]=0
                equiv_dict_ncs_copy_dict[id][value]={}
              equiv_dict[id][value]+=1
              if not n in equiv_dict_ncs_copy_dict[id][value]:
                equiv_dict_ncs_copy_dict[id][value][n]=0
              equiv_dict_ncs_copy_dict[id][value][n]+=1  # how many are ncs copy n
    return duplicate_dict,equiv_dict,equiv_dict_ncs_copy_dict,\
        region_range_dict,region_centroid_dict,region_scattered_points_dict
  
  def get_region_scattered_points_dict(
    edited_volume_list=None,
    edited_mask=None,
    unit_cell=None,
    sampling_rate=None,
    target_points_per_region=None,
    minimum_points_per_region=None,
    maximum_points_per_region=None):
  
    # Get sampled points in each region
    sample_dict={}
    region_scattered_points_dict={} # some points in each region
    if not sampling_rate:
      sampling_rate=edited_volume_list[0]//target_points_per_region
      sampling_rate_set=False
    else:
      sampling_rate_set=True
    volumes=flex.int()
    sampling_rates=flex.int()
    id_list=[]
    # have to set up dummy first set:
    volumes.append(0)
    sampling_rates.append(0)
    id_list.append(0)
  
    for i in range(len(edited_volume_list)):
      id=i+1
      v=edited_volume_list[i]
  
      region_scattered_points_dict[id]=flex.vec3_double()
  
      volumes.append(v)
      if sampling_rate_set:
        sample_dict[id]=sampling_rate
        sampling_rates.append(sampling_rate)
      else:
        sample_dict[id]=max(1,
          max(v//maximum_points_per_region,
            min(v//minimum_points_per_region,
                sampling_rate)  ))
        sampling_rates.append(max(1,
        max(v//maximum_points_per_region,
            min(v//minimum_points_per_region,
                sampling_rate)  )))
      id_list.append(id)
  
    sample_regs_obj = maptbx.sample_all_mask_regions(
        mask=edited_mask,
        volumes=volumes,
        sampling_rates=sampling_rates,
        unit_cell=unit_cell)
  
    for id in id_list[1:]:  # skip the dummy first set
      region_scattered_points_dict[id]=sample_regs_obj.get_array(id)
  
    return region_scattered_points_dict
  
  def remove_bad_regions(params=None,
    duplicate_dict=None,
    edited_volume_list=None,
    out=sys.stdout):
  
    worst_list=[]
    for id in duplicate_dict.keys():
      fract=duplicate_dict[id]/edited_volume_list[id-1]
      if duplicate_dict[id] and fract >=params.segmentation.max_overlap_fraction:
        worst_list.append([fract,id])
      else:
        del duplicate_dict[id]
    worst_list.sort()
    worst_list.reverse()
  
    bad_region_list=[]
    max_number_to_remove=int(0.5+
      0.01*params.segmentation.remove_bad_regions_percent*len(edited_volume_list))
    if worst_list:
      print("\nRegions that span multiple NCS au:", file=out)
      for fract,id in worst_list:
        print("ID: %d  Duplicate points: %d (%.1f %%)" %(
          id,duplicate_dict[id],100.*fract), end=' ', file=out)
        if  len(bad_region_list)<max_number_to_remove:
           bad_region_list.append(id)
           print(" (removed)", file=out)
        else:
           print(file=out)
  
    new_sorted_by_volume=[]
    region_list=[]
    region_volume_dict={}
    for i in range(len(edited_volume_list)):
      id=i+1
      v=edited_volume_list[i]
      new_sorted_by_volume.append([v,id])
      region_list.append(id)
      region_volume_dict[id]=v
    if bad_region_list:
      print("Bad regions (excluded)",bad_region_list, file=out)
    return region_list,region_volume_dict,new_sorted_by_volume,bad_region_list
  
  def sort_by_ncs_overlap(matches,equiv_dict_ncs_copy_dict_id):
      sort_list=[]
      for id1 in matches:
        key,n=top_key(equiv_dict_ncs_copy_dict_id[id1]) # Take top ncs_copy
        sort_list.append([n,id1])
      sort_list.sort()
      sort_list.reverse()
      key_list=[]
      for n,id1 in sort_list:
        key_list.append(id1)
      return key_list
  
  
  def get_ncs_equivalents(
      bad_region_list=None,
      region_list=None,
      region_scattered_points_dict=None,
      equiv_dict=None,
      ncs_copies=None,
      equiv_dict_ncs_copy_dict=None,
      min_coverage=.10,
      out=sys.stdout):
  
    equiv_dict_ncs_copy={}
    for id in region_list:
      if id in bad_region_list: continue
      match_dict=equiv_dict.get(id,{}) # which are matches
      matches=list(match_dict.keys())
      if not matches: continue
      key_list=sort_by_ncs_overlap(matches,equiv_dict_ncs_copy_dict[id])
      n_found=0
      for id1 in key_list:
        #     id matches id1 N=match_dict[id1]
        #  2017-12-16 Do not include if there is a cell translation
  
        key,n=top_key(equiv_dict_ncs_copy_dict[id][id1]) # ncs_copy, n-overlap
        if n<min_coverage*region_scattered_points_dict[id].size():
          break
        else:
          if not id in equiv_dict_ncs_copy:equiv_dict_ncs_copy[id]={}
          equiv_dict_ncs_copy[id][id1]=key
          n_found+=1
          if n_found>=ncs_copies-1:
            break
  
    return equiv_dict_ncs_copy
  
  def get_overlap(l1,l2):
    overlap_list=[]
    l1a=single_list(l1)
    l2a=single_list(l2)
    for i in l1a:
      if i in l2a and not i in overlap_list: overlap_list.append(i)
    return overlap_list
  
  def group_ncs_equivalents(params,
      region_list=None,
      region_volume_dict=None,
      equiv_dict_ncs_copy=None,
      tracking_data=None,
      split_if_possible=None,
      out=sys.stdout):
  
    # equiv_dict_ncs_copy[id][id1]=ncs_copy
    # group together all the regions that are related to region 1...etc
    # if split_if_possible then skip all groups with multiple entries
  
    ncs_equiv_groups_as_list=[]
    ncs_equiv_groups_as_dict={}
    for id in region_list:
      equiv_group={}  #equiv_group[ncs_copy]=[id1,id2,id3...]
      equiv_group[0]=[id] # always
      for id1 in equiv_dict_ncs_copy.get(id,{}).keys():
        ncs_copy=equiv_dict_ncs_copy[id][id1]
        if not ncs_copy in equiv_group: equiv_group[ncs_copy]=[]
        equiv_group[ncs_copy].append(id1) # id1 is ncs_copy of id
      all_single=True
      equiv_group_as_list=[]
      total_grid_points=0
      missing_ncs_copies=[]
      present_ncs_copies=[]
      for ncs_copy in range(tracking_data.input_ncs_info.number_of_operators):
          # goes 0 to ncs_copies-1 (including extra ones if present)
        local_equiv_group=equiv_group.get(ncs_copy,[])
        if local_equiv_group:
          equiv_group_as_list.append(local_equiv_group)
          present_ncs_copies.append(ncs_copy)
          if ncs_copy > 0 and \
            len(local_equiv_group)>1 and len(equiv_group.get(0,[]))==1:
            all_single=False
          for id in equiv_group.get(ncs_copy,[]):
            total_grid_points+=region_volume_dict[id]
        else:
          missing_ncs_copies.append(ncs_copy)
      equiv_group_as_list.sort()
      if tracking_data.input_ncs_info.is_helical_symmetry:
        # complete if we have original_number_of_operators worth
        if (not params.segmentation.require_complete) or \
           len(present_ncs_copies)>= \
           tracking_data.input_ncs_info.original_number_of_operators:
          complete=True
        else:
          complete=False
      else:
        if len(missing_ncs_copies)==0:
          complete=True
        else:
          complete=False
      if complete and \
          (not str(equiv_group_as_list) in ncs_equiv_groups_as_dict or
           total_grid_points>ncs_equiv_groups_as_dict[str(equiv_group_as_list)]) \
          and (all_single or (not split_if_possible)):
        ncs_equiv_groups_as_dict[str(equiv_group_as_list)]=total_grid_points
        ncs_equiv_groups_as_list.append([total_grid_points,equiv_group_as_list])
  
    ncs_equiv_groups_as_list.sort()
    ncs_equiv_groups_as_list.reverse()
  
    # Now remove any group that duplicates a previous group
    # 2015-11-07 allow a member to be in multiple groups though (for example
    #   one that spans several groups because it contains 2 region in other ncs
    #   copies)
    #  Make sure that if there are duplicates they are all in the leading
    #    positions of the list (these must be very big ones as they match 2
    #    regions in other ncs copies)
  
    max_duplicates=tracking_data.input_ncs_info.number_of_operators-1 # not all duplicates
    ncs_group_list=[]
    used_list=[]
    print("All equiv groups:", file=out)
    used_regions=[]
    for total_grid_points,equiv_group_as_list in ncs_equiv_groups_as_list:
      duplicate=False
      n_dup=0
      for equiv_group in equiv_group_as_list:
        for x in equiv_group:
          if x in used_list:
            n_dup+=1
      if n_dup>max_duplicates or n_dup >len(equiv_group_as_list)-1:
        duplicate=True
      if not duplicate and n_dup>0:  # check carefully to make sure that all
        # are leading entries
        for ncs_group in ncs_group_list:
          overlaps=get_overlap(ncs_group,equiv_group_as_list)
          if not overlaps: continue
          overlaps.sort()
          expected_match=single_list(equiv_group_as_list)[:len(overlaps)]
          expected_match.sort()
          if overlaps!=expected_match: # not leading entries
            duplicate=True
            break
  
      if not duplicate:
        #print >>out,"NCS GROUP:",equiv_group_as_list,":",total_grid_points
  
        ncs_group_list.append(equiv_group_as_list)
        for equiv_group in equiv_group_as_list:
          for x in equiv_group:
            if not x in used_list: used_list.append(x)
    print("Total NCS groups: %d" %len(ncs_group_list), file=out)
  
    # Make a dict that lists all ids that are in the same group as region x
    shared_group_dict={}
    for ncs_group in ncs_group_list:
      for group_list in ncs_group:
        for id1 in group_list:
          if not id1 in shared_group_dict: shared_group_dict[id1]=[]
          for other_group_list in ncs_group:
            if other_group_list is group_list:continue
            for other_id1 in other_group_list:
              if not other_id1 in shared_group_dict [id1]:
                shared_group_dict[id1].append(other_id1)
  
    return ncs_group_list,shared_group_dict
  
  def identify_ncs_regions(params,
       sorted_by_volume=None,
       co=None,
       min_b=None,
       max_b=None,
       ncs_obj=None,
       tracking_data=None,
       out=sys.stdout):
  
    # 1.choose top regions to work with
    # 2.remove regions that are in more than one au of the NCS
    # 3.identify groups of regions that are related by NCS
    #  Also note the centers and bounds of each region
  
    # Choose number of top regions to consider
  
    max_regions_to_consider=choose_max_regions_to_consider(params,
      sorted_by_volume=sorted_by_volume,
      ncs_copies=tracking_data.input_ncs_info.original_number_of_operators)
  
    print("\nIdentifying NCS-related regions.Total regions to consider: %d" %(
      max_regions_to_consider), file=out)
    if max_regions_to_consider<1:
      print("\nUnable to identify any NCS regions", file=out)
      return None,tracking_data,None
  
    # Go through all grid points; discard if not in top regions
    #  Renumber regions in order of decreasing size
  
    load_saved_files=False  # set to True to load results from previous run
    dump_files=False        # set to True to dump results and speed up next run
    if not load_saved_files:
      edited_mask,edited_volume_list,original_id_from_id=get_edited_mask(
       sorted_by_volume=sorted_by_volume,
       co=co,
       max_regions_to_consider=max_regions_to_consider,out=out)
      if dump_files:
        from libtbx import easy_pickle
        easy_pickle.dump("edited_mask.pkl",
          [edited_mask,edited_volume_list,original_id_from_id])
    else:
      from libtbx import easy_pickle
      [edited_mask,edited_volume_list,original_id_from_id
          ]=easy_pickle.load("edited_mask.pkl")
      print("Loading edited_mask.pkl", file=out)
  
    # edited_mask contains re-numbered region id's
  
    # Identify duplicate and ncs relationships between regions
    # duplicate_dict[id]= number of duplicates for that region
    # equiv_dict[id][other_id]=number_of points other_id matches
                     #  id through an ncs relationship
    if not load_saved_files:
      duplicate_dict,equiv_dict,equiv_dict_ncs_copy_dict,\
        region_range_dict,region_centroid_dict,\
        region_scattered_points_dict=\
      run_get_duplicates_and_ncs(
        ncs_obj=ncs_obj,
        min_b=min_b,
        max_b=max_b,
        edited_mask=edited_mask,
        original_id_from_id=original_id_from_id,
        edited_volume_list=edited_volume_list,
        max_regions_to_consider=max_regions_to_consider,
        tracking_data=tracking_data,
        out=out)
      # Remove any bad regions
      region_list,region_volume_dict,new_sorted_by_volume,\
        bad_region_list=remove_bad_regions(
      params=params,
      duplicate_dict=duplicate_dict,
      edited_volume_list=edited_volume_list,
      out=out)
      # Identify groups of regions that are ncs-related
      # equiv_dict_ncs_copy[id][id1]=ncs_copy of id that corresponds to id1
      equiv_dict_ncs_copy=get_ncs_equivalents(
      region_list=region_list,
      bad_region_list=bad_region_list,
      region_scattered_points_dict=region_scattered_points_dict,
      equiv_dict=equiv_dict,
      ncs_copies=tracking_data.input_ncs_info.number_of_operators,
      equiv_dict_ncs_copy_dict=equiv_dict_ncs_copy_dict,
      out=out)
      if dump_files:
        from libtbx import easy_pickle
        easy_pickle.dump("save.pkl",[duplicate_dict,equiv_dict,region_range_dict,region_centroid_dict,region_scattered_points_dict,region_list,region_volume_dict,new_sorted_by_volume,bad_region_list,equiv_dict_ncs_copy,tracking_data])
        print("Dumped save.pkl", file=out)
    else:
      from libtbx import easy_pickle
      [duplicate_dict,equiv_dict,region_range_dict,region_centroid_dict,region_scattered_points_dict,region_list,region_volume_dict,new_sorted_by_volume,bad_region_list,equiv_dict_ncs_copy,tracking_data]=easy_pickle.load("save.pkl")
      print("Loaded save.pkl", file=out)
  
    # Group together regions that are ncs-related. Also if one ncs
    #   copy has 2 or more regions linked together, group the other ones.
  
    # each entry in ncs_group_list is a list of regions for each ncs_copy:
    #  e.g.,  [[8], [9, 23], [10, 25], [11, 27], [12, 24], [13, 22], [14, 26]]
    #  May contain elements that are in bad_region_list (to exclude later)
    if not load_saved_files:
      ncs_group_list,shared_group_dict=group_ncs_equivalents(params,
      split_if_possible=params.segmentation.split_if_possible,
      tracking_data=tracking_data,
      region_volume_dict=region_volume_dict,
      region_list=region_list,
      equiv_dict_ncs_copy=equiv_dict_ncs_copy,
      out=out)
      if dump_files:
        from libtbx import easy_pickle
        easy_pickle.dump("group_list.pkl",[ncs_group_list,shared_group_dict])
        print("Dumped to group_list.pkl", file=out)
    else:
      from libtbx import easy_pickle
      [ncs_group_list,shared_group_dict]=easy_pickle.load("group_list.pkl")
      print("Loaded group_list.pkl", file=out)
  
    ncs_group_obj=ncs_group_object(
       ncs_group_list=ncs_group_list,
       shared_group_dict=shared_group_dict,
       ncs_obj=ncs_obj,
       crystal_symmetry=tracking_data.crystal_symmetry,
       edited_mask=edited_mask,
       origin_shift=tracking_data.origin_shift,
       co=co,
       min_b=min_b,
       max_b=max_b,
       equiv_dict=equiv_dict,
       bad_region_list=bad_region_list,
       original_id_from_id=original_id_from_id,
       edited_volume_list=edited_volume_list,
       region_range_dict=region_range_dict,
       region_scattered_points_dict=region_scattered_points_dict,
       region_centroid_dict=region_centroid_dict)
  
    return ncs_group_obj,tracking_data,equiv_dict_ncs_copy
  
  def get_center_list(regions,
      region_centroid_dict=None):
    center_list=[]
    for region in regions:
      center_list.append(region_centroid_dict[region])
    return center_list
  
  def get_average_center(regions,
      region_centroid_dict=None):
    center_list=get_center_list(regions,region_centroid_dict=region_centroid_dict)
    for region in regions:
      center_list.append(region_centroid_dict[region])
    average_center=deepcopy(center_list[0])
    if len(center_list)>1:
      for r in center_list[1:]:
        for i in range(3):
          average_center[i]+=r[i]
      for i in range(3):
        average_center[i]/=len(center_list)
    return average_center
  
  def get_dist(r,s):
    dd=0.
    for i in range(3):
      dd+=(r[i]-s[i])**2
    return dd**0.5
  
  def has_intersection(set1,set2):
    set1a=single_list(set1)
    set2a=single_list(set2)
    for x in set1a:
      if x in set2a:
        return True
    return False
  
  def get_scattered_points_list(other_regions,
         region_scattered_points_dict=None):
    scattered_points_list=flex.vec3_double()
    for x in other_regions:
      scattered_points_list.extend(region_scattered_points_dict[x])
    return scattered_points_list
  
  def get_inter_region_dist_dict(ncs_group_obj=None,
      selected_regions=None,target_scattered_points=None):
    dd={}
    for i in range(len(selected_regions)):
      id=selected_regions[i]
      if not id in dd: dd[id]={}
      test_centers=ncs_group_obj.region_scattered_points_dict[id]
      for j in range(i+1,len(selected_regions)):
        id1=selected_regions[j]
        test_centers1=ncs_group_obj.region_scattered_points_dict[id1]
        dist=get_closest_dist(test_centers,test_centers1)
        dd[id][id1]=dist
        if not id1 in dd: dd[id1]={}
        dd[id1][id]=dist
    return dd
  
  def get_dist_to_first_dict(ncs_group_obj=None,
       selected_regions=None,
       inter_region_dist_dict=None,
       target_scattered_points=None):
  
    # Get distance to region 0 ( or to target_scattered_points if supplied)
    dist_to_first_dict={}
    if target_scattered_points:
      start_region=0
      for x in selected_regions:
        dist_to_first_dict[x]=get_closest_dist(
          ncs_group_obj.region_scattered_points_dict[x],
          target_scattered_points)
    else:
      start_region=1
      x0=selected_regions[0]
      dist_to_first_dict[x0]=0
      for x in selected_regions[1:]:
        dist_to_first_dict[x]=inter_region_dist_dict[x0][x]
    changing=True
    while changing:
      changing=False
      for x in selected_regions[start_region:]:
        for y in selected_regions[start_region:]:
          if x==y: continue
          if dist_to_first_dict[y]<dist_to_first_dict[x] and \
              inter_region_dist_dict[x][y]<dist_to_first_dict[x]:
            dist_to_first_dict[x]=max(
              dist_to_first_dict[y],inter_region_dist_dict[x][y])
            changing=True
    return dist_to_first_dict
  
  def radius_of_gyration_of_vector(xyz):
    return (xyz-xyz.mean()).rms_length()
  
  def get_radius_of_gyration(ncs_group_obj=None,
      selected_regions=None):
    # return radius of gyration of points in selected regions
    centers=flex.vec3_double()
    for s in selected_regions:
      centers.append(ncs_group_obj.region_centroid_dict[s])
    centers=centers-centers.mean()
    return centers.rms_length()
  
  
  def get_closest_neighbor_rms(ncs_group_obj=None,selected_regions=None,
      target_scattered_points=None,verbose=False,out=sys.stdout):
  
    # return rms closest distance of each region center to lowest_numbered region,
    #   allowing sequential tracking taking max of inter-region distances
  
    # XXX can't we save some of this for next time?
  
    inter_region_dist_dict=get_inter_region_dist_dict(ncs_group_obj=ncs_group_obj,
       selected_regions=selected_regions)
    if verbose:
      print("Inter-region distance dict:", file=out)
      keys=list(inter_region_dist_dict.keys())
      keys.sort()
      for key in keys:
        for key2 in inter_region_dist_dict[key].keys():
          print("%s  %s  : %.1f " %(key,key2,inter_region_dist_dict[key][key2]), file=out)
  
    dist_to_first_dict=get_dist_to_first_dict(ncs_group_obj=ncs_group_obj,
       selected_regions=selected_regions,
       inter_region_dist_dict=inter_region_dist_dict,
       target_scattered_points=target_scattered_points)
  
    if verbose:
      print("Distance-to-first dict:", file=out)
      keys=list(dist_to_first_dict.keys())
      keys.sort()
      for key in keys: print("\n %s:  %.1f " %(key,dist_to_first_dict[key]), file=out)
  
    if target_scattered_points:
      start_region=0 # we are getting dist to target_scattered_points
    else:
      start_region=1 # we are getting dist to region 0
  
    rms=0.
    rms_n=0.
    for x in selected_regions[start_region:]:
      dist=dist_to_first_dict[x]
      rms+=dist**2
      rms_n+=1.
    if rms_n>1:
      rms/=rms_n
    rms=rms**0.5
    return rms
  
  
  def get_rms(selected_regions=None,
      region_centroid_dict=None):
    # return rms distance of each region center from average of all others
    rms=0.
    rms_n=0.
    for x in selected_regions:
      other_regions=remove_one_item(selected_regions,item_to_remove=x)
      current_center=get_average_center(other_regions,
         region_centroid_dict=region_centroid_dict)
      test_center=region_centroid_dict[x]
      dist=get_dist(current_center,test_center)
      rms+=dist**2
      rms_n+=1.
    if rms_n>1:
      rms/=rms_n
    return rms**0.5
  
  def single_list(list_of_lists):
    single=[]
    for x in list_of_lists:
      if type(x)==type([1,2,3]):
        single+=single_list(x)
      else:
        single.append(x)
    return single
  
  def get_closest_dist(test_center,target_centers):
    # make sure we have target_centers=vec3_double and not a list,
    #  and vec3_double or tuple for test_center
  
    if type(test_center)==type([1,2,3]):
      test_center=flex.vec3_double(test_center)
    if type(target_centers)==type([1,2,3]):
      target_centers=flex.vec3_double(target_centers)
    if test_center.size()<1 or target_centers.size()<1: return None
    closest_dist=test_center.min_distance_between_any_pair(target_centers)
    return closest_dist
  
  def region_lists_have_ncs_overlap(set1,set2,ncs_group_obj=None,cutoff=0):
    for id1 in set1:
      for id2 in set2:
        if id2 in ncs_group_obj.shared_group_dict.get(id1,[]):
          return True
    return False
  
  def get_effective_radius(ncs_group_obj=None,
      target_scattered_points=None,
      weight_rad_gyr=None,
      selected_regions=None):
    sr=deepcopy(selected_regions)
    sr.sort()
    rad_gyr=get_radius_of_gyration(ncs_group_obj=ncs_group_obj,
       selected_regions=sr)
    rms=get_closest_neighbor_rms(ncs_group_obj=ncs_group_obj,
      target_scattered_points=target_scattered_points,
      selected_regions=sr)
    max_cell_dim=0.
    if ncs_group_obj.max_cell_dim and ncs_group_obj.max_cell_dim > 1.0:
      wrg=weight_rad_gyr*(300/ncs_group_obj.max_cell_dim)  # have a consistent scale
    else:
      wrg=weight_rad_gyr
    effective_radius=(rms+wrg*rad_gyr)/(1.+wrg)
    return effective_radius
  
  def add_neighbors(params,
        selected_regions=None,
        max_length_of_group=None,
        target_scattered_points=None,
        tracking_data=None,
        equiv_dict_ncs_copy=None,
        ncs_group_obj=None,out=sys.stdout):
  
    #   Add neighboring regions on to selected_regions.
    #   Same rules as select_from_seed
  
    selected_regions=single_list(deepcopy(selected_regions))
  
    added_regions=[]
    start_dist=get_effective_radius(ncs_group_obj=ncs_group_obj,
          target_scattered_points=target_scattered_points,
          weight_rad_gyr=params.segmentation.weight_rad_gyr,
          selected_regions=selected_regions)
    delta_dist=params.segmentation.add_neighbors_dist
    max_dist=start_dist+delta_dist
  
    starting_selected_regions=deepcopy(selected_regions)
  
    for x in selected_regions:  # delete, add in alternatives one at a time and
      #  keep all the ok ones
      ncs_groups_to_use=get_ncs_related_regions(
        ncs_group_obj=ncs_group_obj,
        selected_regions=[x],
        include_self=False)
  
      for x in ncs_groups_to_use: # try adding from each group
        if x in selected_regions+added_regions:
          continue
        ncs_group=[[x]]
        current_scattered_points_list=get_scattered_points_list(selected_regions,
          region_scattered_points_dict=ncs_group_obj.region_scattered_points_dict)
  
        for ncs_set in ncs_group: # pick the best ncs_set from this group
          if has_intersection(ncs_group_obj.bad_region_list,ncs_set):
            continue
  
          dist=get_effective_radius(ncs_group_obj=ncs_group_obj,
            target_scattered_points=target_scattered_points,
            weight_rad_gyr=params.segmentation.weight_rad_gyr,
            selected_regions=selected_regions+ncs_set)
  
          if dist <= max_dist:
            added_regions.append(x)
  
    selected_regions=selected_regions+added_regions
    dist=get_effective_radius(ncs_group_obj=ncs_group_obj,
            target_scattered_points=target_scattered_points,
            weight_rad_gyr=params.segmentation.weight_rad_gyr,
            selected_regions=selected_regions)
  
    # Identify all the NCS operators required to map final to starting
    # equiv_dict_ncs_copy[id][id1]=ncs_copy of id that corresponds to id1
    ncs_group=ncs_group_obj.ncs_obj.ncs_groups()[0]
    identity_op=ncs_group.identity_op_id()
    ncs_ops_used=[identity_op]
  
    for id in selected_regions:
      related_regions=get_ncs_related_regions(
        ncs_group_obj=ncs_group_obj,
        selected_regions=[id],
        include_self=False)
      for id1 in selected_regions:
        if not id1 in related_regions: continue
        ncs_copy1=equiv_dict_ncs_copy.get(id,{}).get(id1,None)
        ncs_copy2=equiv_dict_ncs_copy.get(id1,{}).get(id,None)
        for a in [ncs_copy1,ncs_copy2]:
          if a is not None and not a in ncs_ops_used:
              ncs_ops_used.append(a)
    selected_regions.sort()
    ncs_ops_used.sort()
    for x in selected_regions:
      print("GROUP ",x,":",ncs_group_obj.shared_group_dict.get(x,[]), file=out)
  
    return selected_regions,dist,ncs_ops_used
  
  def select_from_seed(params,
        starting_regions,
        target_scattered_points=None,
        max_length_of_group=None,
        ncs_groups_to_use=None,
        tracking_data=None,
        ncs_group_obj=None):
    selected_regions=single_list(deepcopy(starting_regions))
    # do not allow any region in ncs_group_obj.bad_region_list
    # also do not allow any region that is in an ncs-related group to any region
    #  already used.  Use ncs_group_obj.equiv_dict to identify these.
    if not ncs_groups_to_use:
      ncs_groups_to_use=ncs_group_obj.ncs_group_list
  
    for ncs_group in ncs_groups_to_use: # try adding from each group
      if max_length_of_group is not None and \
         len(selected_regions)>=max_length_of_group:
        break
      best_ncs_set=None
      best_dist=None
      if has_intersection(ncs_group,selected_regions):
        continue
      current_scattered_points_list=get_scattered_points_list(selected_regions,
         region_scattered_points_dict=ncs_group_obj.region_scattered_points_dict)
      if target_scattered_points:
        current_scattered_points_list.extend(target_scattered_points)
  
      for ncs_set in ncs_group: # pick the best ncs_set from this group
        if has_intersection(ncs_group_obj.bad_region_list,ncs_set): continue
  
        # does any ncs copy of anything in selected_regions actually overlap
        #  with any member of ncs_set... might be efficient to delete the entire
        #   ncs_group if any ncs_set overlaps, but could lose some.
        if region_lists_have_ncs_overlap(ncs_set,selected_regions,
            ncs_group_obj=ncs_group_obj):
          continue
  
        dist=get_effective_radius(ncs_group_obj=ncs_group_obj,
          target_scattered_points=target_scattered_points,
          weight_rad_gyr=params.segmentation.weight_rad_gyr,
          selected_regions=selected_regions+ncs_set)
  
        if best_dist is None or dist<best_dist:
          best_dist=dist
          best_ncs_set=ncs_set
      if best_ncs_set is not None:
        selected_regions+=best_ncs_set
  
    dist=get_effective_radius(ncs_group_obj=ncs_group_obj,
      target_scattered_points=target_scattered_points,
      weight_rad_gyr=params.segmentation.weight_rad_gyr,
      selected_regions=selected_regions)
  
    return selected_regions,dist
  
  def remove_one_item(input_list,item_to_remove=None):
    new_list=[]
    for item in input_list:
      if item != item_to_remove:
        new_list.append(item)
    return new_list
  
  
  def get_ncs_related_regions_specific_list(
      ncs_group_obj=None,
      target_regions=None,
      include_self=False):
    all_regions=[]
    for target_region in target_regions:
      all_regions+=get_ncs_related_regions_specific_target(
        ncs_group_obj=ncs_group_obj,
        target_region=target_region,
        other_regions=remove_one_item(
           target_regions,item_to_remove=target_region),
        include_self=include_self)
    return all_regions
  
  def get_ncs_related_regions_specific_target(
            ncs_group_obj=None,
            target_region=None,
            other_regions=None,
            include_self=False):
    # similar to get_ncs_related_regions, but find just one  ncs group that
    #  contains x but does not contain any member of other_regions
    for ncs_group in ncs_group_obj.ncs_group_list: # might this be the group
      ids_in_group=single_list(ncs_group)
      if not target_region in ids_in_group: continue # does not contain target
      contains_others=False
      for other_id in other_regions:
        if other_id in ids_in_group:
          contains_other=True
          break# contains other members
      if not contains_others:
        # this is the group
        if include_self:
          return ids_in_group
        else:
          return remove_one_item(ids_in_group,item_to_remove=target_region)
    return []
  
  
  def get_ncs_related_regions(
      ncs_group_obj=None,
      selected_regions=None,
      include_self=False):
    # returns a simple list of region ids
    # if include_self then include selected regions and all ncs-related
    #   otherwise do not include selected regions or anything that might
    #   overlap with them
  
    ncs_related_regions=[]
    if include_self:
      for id in selected_regions:
        if not id in ncs_related_regions:
          ncs_related_regions.append(id)
        for ncs_group in ncs_group_obj.ncs_group_list:
          ids_in_group=single_list(ncs_group)
          if id in ids_in_group:  # this group contains this selected id
            for i in ids_in_group:
              if not i in ncs_related_regions:
                ncs_related_regions.append(i)
  
    else:
      for id in selected_regions:
        found=False
        for ncs_group in ncs_group_obj.ncs_group_list:
          ids_in_group=single_list(ncs_group)
          if id in ids_in_group:  # this group contains this selected id
            found=True
            for i in ids_in_group:
              if (not i==id) and (not i in selected_regions) and \
                 (not i in ncs_related_regions):
                ncs_related_regions.append(i)
            break # don't look at any more ncs groups
  
    return ncs_related_regions
  
  def all_elements_are_length_one(list_of_elements):
    for x in list_of_elements:
      if type(x)==type([1,2,3]):
        if len(x)!=1: return False
    return True
  
  def as_list_of_lists(ll):
    new_list=[]
    for x in ll:
      new_list.append([x])
    return new_list
  
  def select_regions_in_au(params,
       ncs_group_obj=None,
       target_scattered_points=None,
       unique_expected_regions=None,
       equiv_dict_ncs_copy=None,
       tracking_data=None,
       out=sys.stdout):
    # Choose one region or set of regions from each ncs_group
    # up to about unique_expected_regions
    # Optimize closeness of centers...
    # If target scattered_points is supplied, include them as allowed target
  
    if not ncs_group_obj.ncs_group_list:
      return ncs_group_obj,[]
  
    max_length_of_group=max(1,unique_expected_regions*
       params.segmentation.max_per_au_ratio)
    print("Maximum length of group: %d" %(max_length_of_group), file=out)
  
    if all_elements_are_length_one(ncs_group_obj.ncs_group_list):
      # This is where there is no ncs. Basically skipping everything
      best_selected_regions=single_list(ncs_group_obj.ncs_group_list)
      best_rms=None
    else:
      #--------------  Find initial set of regions --------------------
      # Seed with members of the first NCS group or with the target points
      #  and find the member of each NCS group that is closest
  
      if target_scattered_points:
        starting_regions=[None]
      else:
        starting_regions=ncs_group_obj.ncs_group_list[0]
  
      best_selected_regions=None
      best_rms=None
      ok_seeds_examined=0
      for starting_region in starting_regions: # NOTE starting_region is a list
        if not starting_region and not target_scattered_points:continue
        if ok_seeds_examined >= params.segmentation.seeds_to_try:
          break # don't bother to keep trying
        if starting_region and starting_region in ncs_group_obj.bad_region_list:
          continue # do not use
        if starting_region: # NOTE: starting_region is a list itself
          starting_region_list=[starting_region]
        else:
          starting_region_list=[]
        selected_regions,rms=select_from_seed(params,
          starting_region_list,
          target_scattered_points=target_scattered_points,
          max_length_of_group=max_length_of_group,
          tracking_data=tracking_data,
          ncs_group_obj=ncs_group_obj)
        if not selected_regions:
          continue
        ok_seeds_examined+=1
        if best_rms is None or rms<best_rms:
          best_rms=rms
          best_selected_regions=selected_regions
          print("New best selected: rms: %7.1f: %s " %(
             rms,str(selected_regions)), file=out)
  
      if best_rms is not None:
        print("Best selected so far: rms: %7.1f: %s " %(
              best_rms,str(best_selected_regions)), file=out)
  
      if not best_selected_regions:
        print("\nNo NCS regions found ...", file=out)
        return ncs_group_obj,[]
  
      # Now we have a first version of best_rms, best_selected_regions
  
      #--------------  END Find initial set of regions --------------------
  
  
      #--------------  Optimize choice of regions -------------------------
      max_tries=10
      improving=True
      itry=0
      while improving and itry<max_tries:
        itry+=1
        improving=False
        previous_selected_regions=deepcopy(best_selected_regions)
        previous_selected_regions.sort()
        print("\nTry %d for optimizing regions" %(itry), file=out)
        # Now see if replacing any regions with alternatives would improve it
        for x in previous_selected_regions:
          starting_regions=remove_one_item(previous_selected_regions,
            item_to_remove=x)
          # identify ncs_related regions to x, but not to other members of
          #  selected_regions
          ncs_related_regions=get_ncs_related_regions_specific_list(
            ncs_group_obj=ncs_group_obj,
            include_self=True,
            target_regions=[x])
          if not ncs_related_regions: continue
          ncs_groups_to_use=[as_list_of_lists(ncs_related_regions)]
          new_selected_regions,rms=select_from_seed(params,starting_regions,
            target_scattered_points=target_scattered_points,
            max_length_of_group=max_length_of_group,
            tracking_data=tracking_data,
            ncs_groups_to_use=ncs_groups_to_use,
            ncs_group_obj=ncs_group_obj)
  
          if not new_selected_regions: continue
          if best_rms is None or rms<best_rms:
            best_selected_regions=new_selected_regions
            best_selected_regions.sort()
            best_rms=rms
            improving=True
        print("Optimized best selected: rms: %7.1f: %s " %(
            best_rms,str(best_selected_regions)), file=out)
  
        # Done with this try
  
    selected_regions=best_selected_regions
    selected_regions.sort()
    if params.map_modification.regions_to_keep:
      selected_regions=selected_regions[:params.map_modification.regions_to_keep]
  
    rms=get_closest_neighbor_rms(ncs_group_obj=ncs_group_obj,
      selected_regions=selected_regions,verbose=False,out=out)
  
    if params.segmentation.add_neighbors and \
         ncs_group_obj.ncs_obj.max_operators()>1:
      print("\nAdding neighbor groups...", file=out)
      selected_regions,rms,ncs_ops_used=add_neighbors(params,
            selected_regions=selected_regions,
            max_length_of_group=max_length_of_group,
            target_scattered_points=target_scattered_points,
            equiv_dict_ncs_copy=equiv_dict_ncs_copy,
            tracking_data=tracking_data,
            ncs_group_obj=ncs_group_obj,out=out)
    else:
      ncs_ops_used=None
  
    print("\nFinal selected regions with rms of %6.2f: " %(rms), end=' ', file=out)
    for x in selected_regions:
      print(x, end=' ', file=out)
    if ncs_ops_used:
      print("\nNCS operators used: ", end=' ', file=out)
      for op in ncs_ops_used:  print(op, end=' ', file=out)
      print(file=out)
    # Save an ncs object containing just the ncs_ops_used
    ncs_group_obj.set_ncs_ops_used(ncs_ops_used)
  
    # Identify scattered points for all selected regions:
  
    scattered_points=get_scattered_points_list(selected_regions,
       region_scattered_points_dict=ncs_group_obj.region_scattered_points_dict)
  
    # Identify ncs-related regions for all the selected regions
    self_and_ncs_related_regions=get_ncs_related_regions(
      ncs_group_obj=ncs_group_obj,
      selected_regions=selected_regions,
      include_self=True)
  
    ncs_related_regions=get_ncs_related_regions(
      ncs_group_obj=ncs_group_obj,
      selected_regions=selected_regions,
      include_self=False)
  
    print("NCS-related regions (not used): %d " %(len(ncs_related_regions)), file=out)
    ncs_group_obj.set_selected_regions(selected_regions)
    ncs_group_obj.set_self_and_ncs_related_regions(self_and_ncs_related_regions)
    ncs_group_obj.set_ncs_related_regions(ncs_related_regions)
  
    return ncs_group_obj,scattered_points
  
  def get_bool_mask_as_int(ncs_group_obj=None,mask_as_int=None,mask_as_bool=None):
    if mask_as_int:
      mask_as_int=mask_as_int.deep_copy()
    else:
      mask_as_int=ncs_group_obj.edited_mask.deep_copy()
    s = (mask_as_bool==True)
    mask_as_int = mask_as_int.set_selected(s,1)
    mask_as_int = mask_as_int.set_selected(~s,0)
    return mask_as_int
  
  def get_bool_mask_of_regions(ncs_group_obj=None,region_list=None,
      expand_size=None):
    s = (ncs_group_obj.edited_mask == -1)
    if region_list is None: region_list=[]
    for id in region_list:
  
      if not expand_size:
        s |= (ncs_group_obj.edited_mask==id)  # just take this region
  
      else:  # expand the size of the regions...use expand_mask which operates
           # on the original id numbers and uses the co
        bool_region_mask = ncs_group_obj.co.expand_mask(
          id_to_expand=ncs_group_obj.original_id_from_id[id],
          expand_size=expand_size)
        s |= (bool_region_mask== True)
  
    bool_mask = ncs_group_obj.co.expand_mask(id_to_expand=1,expand_size=1) # just to get bool mask
    bool_mask = bool_mask.set_selected(s,True)
    bool_mask = bool_mask.set_selected(~s,False)
  
    return bool_mask
  
  def create_remaining_mask_and_map(params,
      ncs_group_obj=None,
      map_data=None,
      crystal_symmetry=None,
      out=sys.stdout):
  
    if not ncs_group_obj.selected_regions:
      print("No regions selected", file=out)
      return map_data
  
    # create new remaining_map containing everything except the part that
    # has been interpreted (and all points in interpreted NCS-related copies)
  
    bool_all_used=get_bool_mask_of_regions(ncs_group_obj=ncs_group_obj,
     region_list=ncs_group_obj.selected_regions+
         ncs_group_obj.self_and_ncs_related_regions,
     expand_size=params.segmentation.expand_size)
    map_data_remaining=map_data.deep_copy()
    s=(bool_all_used==True)
  
    map_data_remaining=map_data_remaining.set_selected(s,
      params.segmentation.value_outside_mask)
    return map_data_remaining
  
  def get_lower(lower_bounds,lower):
    new_lower=[]
    for i in range(3):
      if lower_bounds[i] is None:
        new_lower.append(lower[i])
      elif lower[i] is None:
        new_lower.append(lower_bounds[i])
      else:
        new_lower.append(min(lower_bounds[i],lower[i]))
    return new_lower
  
  def get_upper(upper_bounds,upper):
    new_upper=[]
    for i in range(3):
      if upper_bounds[i] is None:
        new_upper.append(upper[i])
      elif upper[i] is None:
        new_upper.append(upper_bounds[i])
      else:
        new_upper.append(max(upper_bounds[i],upper[i]))
    return new_upper
  
  def get_bounds(ncs_group_obj=None,id=None):
    orig_id=ncs_group_obj.original_id_from_id[id]
    lower=ncs_group_obj.min_b[orig_id]
    upper=ncs_group_obj.max_b[orig_id]
    return lower,upper
  
  def get_selected_and_related_regions(params,
      ncs_group_obj=None):
    # Identify all points in the targeted regions
    bool_selected_regions=get_bool_mask_of_regions(ncs_group_obj=ncs_group_obj,
       region_list=ncs_group_obj.selected_regions,
       expand_size=params.segmentation.expand_size+\
        params.segmentation.mask_additional_expand_size)
    # and all points in NCS-related copies (to be excluded)
    if params.segmentation.exclude_points_in_ncs_copies and (
       not params.segmentation.add_neighbors):
      bool_ncs_related_mask=get_bool_mask_of_regions(ncs_group_obj=ncs_group_obj,
         region_list=ncs_group_obj.ncs_related_regions)
       # NOTE: using ncs_related_regions here NOT self_and_ncs_related_regions
    else:
      bool_ncs_related_mask=None
  
    lower_bounds=[None,None,None]
    upper_bounds=[None,None,None]
    if ncs_group_obj.selected_regions:
      for id in ncs_group_obj.selected_regions:
        lower,upper=get_bounds(
          ncs_group_obj=ncs_group_obj,id=id)
        lower_bounds=get_lower(lower_bounds,lower)
        upper_bounds=get_upper(upper_bounds,upper)
  
    return bool_selected_regions,bool_ncs_related_mask,lower_bounds,upper_bounds
  
  def adjust_bounds(params,
     lower_bounds,upper_bounds,map_data=None,out=sys.stdout):
    # range is lower_bounds to upper_bounds
    lower_bounds=list(lower_bounds)
    upper_bounds=list(upper_bounds)
    if params is None or params.output_files.box_buffer is None:
       box_buffer=0
    else:
       box_buffer=params.output_files.box_buffer
    for i in range(3):
      if lower_bounds[i] is None: lower_bounds[i]=0
      if upper_bounds[i] is None: upper_bounds[i]=0
      lower_bounds[i]-=box_buffer
      lower_bounds[i]=max(0,lower_bounds[i])
      upper_bounds[i]+=box_buffer
      upper_bounds[i]=min(map_data.all()[i]-1,upper_bounds[i])
  
  
    """
    print >>out,"\nRange:  X:(%6d,%6d)    Y:(%6d,%6d)    Z:(%6d,%6d)" %(
       lower_bounds[0],upper_bounds[0],
       lower_bounds[1],upper_bounds[1],
       lower_bounds[2],upper_bounds[2])
    """
  
    return lower_bounds,upper_bounds
  
  def write_region_maps(params,
      ncs_group_obj=None,
      map_data=None,
      tracking_data=None,
      remainder_ncs_group_obj=None,
      regions_to_skip=None,
      out=sys.stdout):
    remainder_regions_written=[]
    map_files_written=[]
    if not ncs_group_obj:
      return map_files_written,remainder_regions_written
  
    if not ncs_group_obj.selected_regions:
      return map_files_written,remainder_regions_written
  
    for id in ncs_group_obj.selected_regions:
  
  
      if regions_to_skip and id in regions_to_skip:
        print("Skipping remainder region %d (already written out)" %(id), file=out)
        continue
      print("Writing region %d" %(id), end=' ', file=out)
  
      # dummy atoms representing this region
      sites=ncs_group_obj.region_scattered_points_dict[id]
  
      bool_region_mask = ncs_group_obj.co.expand_mask(
          id_to_expand=ncs_group_obj.original_id_from_id[id],
          expand_size=params.segmentation.expand_size)
  
      s = (bool_region_mask==True)
  
      lower_bounds,upper_bounds=get_bounds(ncs_group_obj=ncs_group_obj,id=id)
  
      if remainder_ncs_group_obj:
        for remainder_id in remainder_ncs_group_obj.remainder_id_dict.keys():
          if remainder_ncs_group_obj.remainder_id_dict[remainder_id]==id:
            remainder_regions_written.append(remainder_id)
  
            sites.extend(
              remainder_ncs_group_obj.region_scattered_points_dict[remainder_id])
  
            print("(including remainder region %d)" %(remainder_id), end=' ', file=out)
            remainder_bool_region_mask = remainder_ncs_group_obj.co.expand_mask(
             id_to_expand=remainder_ncs_group_obj.original_id_from_id[remainder_id],
             expand_size=params.segmentation.expand_size)
            s|= (remainder_bool_region_mask==True)
            lower,upper=get_bounds(
              ncs_group_obj=remainder_ncs_group_obj,id=remainder_id)
            lower_bounds=get_lower(lower_bounds,lower)
            upper_bounds=get_upper(upper_bounds,upper)
  
  
      region_mask = map_data.deep_copy()
      region_mask = region_mask.set_selected(s,1)
      region_mask = region_mask.set_selected(~s,0)
      local_map_data=map_data.deep_copy()
      local_map_data=local_map_data * region_mask.as_double()
  
      # Now cut down the map to the size we want
      lower_bounds,upper_bounds=adjust_bounds(params,lower_bounds,upper_bounds,
        map_data=map_data,out=out)
      box_map,box_crystal_symmetry,\
        dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
        map_data=local_map_data, \
         crystal_symmetry=tracking_data.crystal_symmetry,
         min_point=lower_bounds, max_point=upper_bounds,out=out)
  
      if remainder_ncs_group_obj:
        text=""
      else:
        text="_r"
      base_file='map%s_%d.ccp4' %(text, id)
      base_pdb_file='atoms%s_%d.pdb' %(text, id)
      if tracking_data.params.output_files.output_directory:
        if not os.path.isdir(tracking_data.params.output_files.output_directory):
          os.mkdir(tracking_data.params.output_files.output_directory)
        file_name=os.path.join(tracking_data.params.output_files.output_directory,base_file)
        pdb_file_name=os.path.join(
          tracking_data.params.output_files.output_directory,base_pdb_file)
      else:
        file_name=base_file
        pdb_file_name=base_pdb_file
      write_ccp4_map(box_crystal_symmetry,file_name, box_map)
      print("to %s" %(file_name), file=out)
      map_files_written.append(file_name)
  
      tracking_data.add_output_region_map_info(
        file_name=file_name,
        crystal_symmetry=box_crystal_symmetry,
        origin=box_map.origin(),
        all=box_map.all(),
        map_id=base_file)
  
      print("Atoms representation written to %s" %(pdb_file_name), file=out)
      write_atoms(tracking_data=tracking_data,sites=sites,file_name=pdb_file_name,
         out=out)
      tracking_data.add_output_region_pdb_info(
        file_name=pdb_file_name)
  
    return map_files_written,remainder_regions_written
  
  def get_bounds_from_sites(sites_cart=None,map_data=None,
      unit_cell=None):
    lower_bounds=[None,None,None]
    upper_bounds=[None,None,None]
    sites_frac=unit_cell.fractionalize(sites_cart)
    nx,ny,nz=map_data.all()
    for x_frac in sites_frac:
      x=[
        int(0.5+nx*x_frac[0]),
        int(0.5+ny*x_frac[1]),
        int(0.5+nz*x_frac[2])]
  
      if lower_bounds[0] is None or x[0]<lower_bounds[0]: lower_bounds[0]=x[0]
      if lower_bounds[1] is None or x[1]<lower_bounds[1]: lower_bounds[1]=x[1]
      if lower_bounds[2] is None or x[2]<lower_bounds[2]: lower_bounds[2]=x[2]
  
      if upper_bounds[0] is None or x[0]>upper_bounds[0]: upper_bounds[0]=x[0]
      if upper_bounds[1] is None or x[1]>upper_bounds[1]: upper_bounds[1]=x[1]
      if upper_bounds[2] is None or x[2]>upper_bounds[2]: upper_bounds[2]=x[2]
    return lower_bounds,upper_bounds
  
  def write_output_files(params,
      tracking_data=None,
      map_data=None,
      half_map_data_list=None,
      ncs_group_obj=None,
      remainder_ncs_group_obj=None,
      pdb_hierarchy=None,
      removed_ncs=None,
      out=sys.stdout):
  
    half_map_data_list_au=[]
    if not half_map_data_list: half_map_data_list=[]
  
    if params.output_files.au_output_file_stem:
      au_mask_output_file=os.path.join(tracking_data.params.output_files.output_directory,params.output_files.au_output_file_stem+"_mask.ccp4")
      au_map_output_file=os.path.join(tracking_data.params.output_files.output_directory,params.output_files.au_output_file_stem+"_map.ccp4")
      au_atom_output_file=os.path.join(tracking_data.params.output_files.output_directory,params.output_files.au_output_file_stem+"_atoms.pdb")
    else:
      au_mask_output_file=None
      au_map_output_file=None
      au_atom_output_file=None
  
    # Write out pdb file with dummy atoms for the AU to au_atom_output_file
    if au_atom_output_file and params.output_files.write_output_maps:
      sites=flex.vec3_double()
      for id in ncs_group_obj.selected_regions:
        sites.extend(ncs_group_obj.region_scattered_points_dict[id])
      if remainder_ncs_group_obj:
        for id in remainder_ncs_group_obj.selected_regions:
          sites.extend(remainder_ncs_group_obj.region_scattered_points_dict[id])
      write_atoms(tracking_data=tracking_data,sites=sites,
        file_name=au_atom_output_file,out=out)
      tracking_data.set_output_ncs_au_pdb_info(file_name=au_atom_output_file)
  
  
    # Write out mask and map representing one NCS copy and none of
    #   other NCS copies.  Expand the mask to include neighboring points (but
    #   not those explicitly in other NCS copies
  
    if params.map_modification.soft_mask and params.control.save_box_map_ncs_au:
      mask_expand_size=estimate_expand_size(
         crystal_symmetry=tracking_data.crystal_symmetry,
         map_data=map_data,
         expand_target=tracking_data.params.segmentation.mask_expand_ratio*\
            tracking_data.params.crystal_info.resolution,
            out=out)
      params.segmentation.mask_additional_expand_size = max(mask_expand_size,
        params.segmentation.mask_additional_expand_size,)
  
    bool_selected_regions,bool_ncs_related_mask,lower_bounds,upper_bounds=\
       get_selected_and_related_regions(
        params,ncs_group_obj=ncs_group_obj)
    if bool_ncs_related_mask is not None:
      s_ncs_related =  (bool_ncs_related_mask==True)
    else:
      s_ncs_related =  None
  
    # Add in remainder regions if present
    if remainder_ncs_group_obj:
      bool_remainder_selected_regions,bool_remainder_ncs_related_mask,\
        remainder_lower_bounds,remainder_upper_bounds=\
         get_selected_and_related_regions(
         params,ncs_group_obj=remainder_ncs_group_obj)
  
      lower_bounds=get_lower(lower_bounds,remainder_lower_bounds)
      upper_bounds=get_upper(upper_bounds,remainder_upper_bounds)
  
      s_remainder_au =  (bool_remainder_selected_regions==True)
      bool_selected_regions=bool_selected_regions.set_selected(
         s_remainder_au,True)
      if s_ncs_related is not None and \
           bool_remainder_ncs_related_mask is not None:
        s_ncs_related |=  (bool_remainder_ncs_related_mask==True)
  
    # Now create NCS mask by eliminating all points in target (expanded) in
    #   NCS-related copies
    if s_ncs_related is not None:
      bool_selected_regions=bool_selected_regions.set_selected(
         s_ncs_related,False)
  
    if tracking_data.params.map_modification.regions_to_keep is None:
      # Identify full (possibly expanded) ncs au starting with what we have
      au_mask=get_one_au(tracking_data=tracking_data,
         starting_mask=bool_selected_regions,
        removed_ncs=removed_ncs,
        ncs_obj=ncs_group_obj.ncs_obj,map_data=map_data,out=out)
      print("\nExpanding NCS AU if necessary...", file=out)
      print("Size of AU mask: %s  Current size of AU: %s" %(
        au_mask.count(True),bool_selected_regions.count(True)), file=out)
      bool_selected_regions=(bool_selected_regions | au_mask)
      print("New size of AU mask: %s" %(bool_selected_regions.count(True)), file=out)
  
    sites_cart=get_marked_points_cart(mask_data=bool_selected_regions,
       unit_cell=ncs_group_obj.crystal_symmetry.unit_cell(),
       every_nth_point=tracking_data.params.segmentation.grid_spacing_for_au,
       boundary_radius=tracking_data.params.segmentation.radius)
    sites_lower_bounds,sites_upper_bounds=get_bounds_from_sites(
        unit_cell=ncs_group_obj.crystal_symmetry.unit_cell(),
        sites_cart=sites_cart,map_data=map_data)
    print("Original bounds: %5s  %5s  %5s  to %5s  %5s  %5s" %(
      tuple(lower_bounds+upper_bounds)), file=out)
    lower_bounds=get_lower(lower_bounds,sites_lower_bounds)
    upper_bounds=get_upper(upper_bounds,sites_upper_bounds)
    print("Updated bounds:  %5s  %5s  %5s  to %5s  %5s  %5s" %(
      tuple(lower_bounds+upper_bounds)), file=out)
  
    lower_bounds,upper_bounds=adjust_bounds(params,lower_bounds,upper_bounds,
      map_data=map_data,out=out)
    box_ncs_au=params.segmentation.box_ncs_au
    if (not box_ncs_au):
      print("Using entire input map (box_ncs_au=False)", file=out)
      lower_bounds=map_data.origin()
      upper_bounds=tuple(matrix.col(map_data.all())+
          matrix.col(map_data.origin())-matrix.col((1,1,1)))
  
  
    print("\nMaking two types of maps for AU of NCS mask and map with "+\
        "buffer of %d grid units \nin each direction around AU" %(
        params.output_files.box_buffer), file=out)
    if params.output_files.write_output_maps:
     print("Both types of maps have the same origin and overlay on %s" %(
     os.path.join(tracking_data.params.output_files.output_directory,
       params.output_files.shifted_map_file)), file=out)
  
  
     print("\nThe standard maps (%s, %s) have the \noriginal cell dimensions." %(
     os.path.join(tracking_data.params.output_files.output_directory,au_mask_output_file),
     os.path.join(tracking_data.params.output_files.output_directory,au_map_output_file))+\
     "\nThese maps show only the unique (NCS AU) part of the map.", file=out)
  
     print("\nThe cut out box_maps (%s, %s) have \nsmaller cell dimensions." %(
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_mask_file),
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_map_file),) +\
     "\nThese maps also show only the unique part of the map and have this"+\
     "\nunique part cut out.\n", file=out)
  
  
    # Write out NCS AU with shifted origin but initial crystal_symmetry
    # Mask
    mask_data_ncs_au=get_bool_mask_as_int(
       ncs_group_obj=ncs_group_obj,mask_as_bool=bool_selected_regions)
  
    if au_mask_output_file and params.output_files.write_output_maps:
      # Write out the mask (as int)
      write_ccp4_map(tracking_data.crystal_symmetry,
        au_mask_output_file,mask_data_ncs_au)
      print("Output NCS AU mask:  %s" %(au_mask_output_file), file=out)
      tracking_data.set_output_ncs_au_mask_info(
        file_name=au_mask_output_file,
        crystal_symmetry=tracking_data.crystal_symmetry,
        origin=mask_data_ncs_au.origin(),
        all=mask_data_ncs_au.all())
  
    # Map
    map_data_ncs_au=map_data.deep_copy()
    s=(bool_selected_regions==True)
    mask=map_data.deep_copy()
    mask=mask.set_selected(s,1)
    mask=mask.set_selected(~s,0)
    if params.map_modification.soft_mask:
      # buffer and smooth the mask
      print("Smoothing mask")
      map_data_ncs_au,smoothed_mask_data=apply_soft_mask(map_data=map_data_ncs_au,
        mask_data=mask.as_double(),
        rad_smooth=tracking_data.params.crystal_info.resolution,
        crystal_symmetry=tracking_data.crystal_symmetry,
        out=out)
      half_map_data_list_au=[]
      for hm in half_map_data_list:  # apply mask to half maps
        hm_data_ncs_au,hm_smoothed_mask_data=apply_soft_mask(
          map_data=hm.deep_copy().as_double(),
          mask_data=mask.as_double(),
          rad_smooth=tracking_data.params.crystal_info.resolution,
          crystal_symmetry=tracking_data.crystal_symmetry,
          out=out)
        half_map_data_list_au.append(hm_data_ncs_au)
  
  
    elif (box_ncs_au): # usual.  If box_ncs_au is False, do not mask
  
      map_data_ncs_au=map_data_ncs_au*mask
  
      one_d=map_data_ncs_au.as_1d()
      n_zero=mask.count(0)
      n_tot=mask.size()
      mean_in_box=one_d.min_max_mean().mean*n_tot/(n_tot-n_zero)
      map_data_ncs_au=map_data_ncs_au+(1-mask)*mean_in_box
      half_map_data_list_au=[]
      for hm in half_map_data_list:  # apply mask to half maps
        one_d=hm.as_1d()
        mean_in_box=one_d.min_max_mean().mean*n_tot/(n_tot-n_zero)
        hm_data_ncs_au=hm+(1-mask)*mean_in_box
        half_map_data_list_au.append(hm_data_ncs_au)
  
      del one_d,mask
  
    if au_map_output_file and params.output_files.write_output_maps:
      # Write out the NCS au of density
      write_ccp4_map(tracking_data.crystal_symmetry,au_map_output_file,
        map_data_ncs_au)
      print("Output NCS AU map:  %s" %(au_map_output_file), file=out)
      tracking_data.set_output_ncs_au_map_info(
        file_name=au_map_output_file,
        crystal_symmetry=tracking_data.crystal_symmetry,
        origin=map_data_ncs_au.origin(),
        all=map_data_ncs_au.all())
  
    # Now box_map of cut out AU
  
    box_mask_ncs_au,box_crystal_symmetry,\
          dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
         map_data=mask_data_ncs_au.as_double(),
         crystal_symmetry=tracking_data.crystal_symmetry,
         min_point=lower_bounds, max_point=upper_bounds,out=out)
  
    # Mask
    if params.output_files.box_mask_file and params.output_files.write_output_maps:
      # write out box_map NCS mask representing one AU of the NCS
      write_ccp4_map(
       box_crystal_symmetry,
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_mask_file),
        box_mask_ncs_au)
      print("Output NCS au as box (cut out) mask:  %s " %(
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_mask_file)), file=out)
      tracking_data.set_output_box_mask_info(
        file_name=os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_mask_file),
        crystal_symmetry=box_crystal_symmetry,
        origin=box_mask_ncs_au.origin(),
        all=box_mask_ncs_au.all())
  
    # Map
    box_map_ncs_au,box_crystal_symmetry,\
         dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
         soft_mask=tracking_data.params.map_modification.soft_mask,
         resolution=tracking_data.params.crystal_info.resolution,
         map_data=map_data_ncs_au.as_double(),
         crystal_symmetry=tracking_data.crystal_symmetry,
         min_point=lower_bounds, max_point=upper_bounds,out=out)
  
    half_map_data_list_au_box=[]
    for hmdlu in half_map_data_list_au:
      hm_box_map_ncs_au,dummy_box_crystal_symmetry,\
         dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
         soft_mask=tracking_data.params.map_modification.soft_mask,
         resolution=tracking_data.params.crystal_info.resolution,
         map_data=hmdlu.as_double(),
         crystal_symmetry=tracking_data.crystal_symmetry,
         min_point=lower_bounds, max_point=upper_bounds,out=out)
      half_map_data_list_au_box.append(hm_box_map_ncs_au)
  
    if params.control.save_box_map_ncs_au:
         tracking_data.set_box_map_ncs_au_map_data(
         box_map_ncs_au_crystal_symmetry=box_crystal_symmetry,
         box_map_ncs_au_map_data=box_map_ncs_au,
         box_map_ncs_au_half_map_data_list=half_map_data_list_au_box,
         )
  
    write_ccp4_map(tracking_data.crystal_symmetry,'map_data_ncs_au.ccp4',map_data_ncs_au)
    write_ccp4_map(box_crystal_symmetry,'box_map_ncs_au.ccp4',box_map_ncs_au)
  
    if params.output_files.box_map_file:
      # write out NCS map as box_map (cut out region of map enclosed in box_mask)
      if params.output_files.write_output_maps:
        write_ccp4_map(box_crystal_symmetry,
          os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_map_file),
          box_map_ncs_au)
        print("Output NCS au as box (cut out) map:  %s " %(
        os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_map_file)), file=out)
        tracking_data.set_output_box_map_info(
          file_name=os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_map_file),
          crystal_symmetry=box_crystal_symmetry,
          origin=box_map_ncs_au.origin(),
          all=box_map_ncs_au.all())
  
  
    # Write out all the selected regions
    if params.output_files.write_output_maps:
      print("\nWriting out region maps. "+\
        "These superimpose on the NCS AU map \nand "+\
        "mask %s,%s\n" %(
          os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_map_file),
          os.path.join(tracking_data.params.output_files.output_directory,params.output_files.box_mask_file),), file=out)
  
      map_files_written,remainder_regions_written=write_region_maps(params,
        map_data=map_data,
        tracking_data=tracking_data,
        ncs_group_obj=ncs_group_obj,
        remainder_ncs_group_obj=remainder_ncs_group_obj,
        out=out)
  
      # and pick up the remainder regions not already written
      remainder_map_files_written,dummy_remainder=write_region_maps(params,
        map_data=map_data,
        tracking_data=tracking_data,
        ncs_group_obj=remainder_ncs_group_obj,
        regions_to_skip=remainder_regions_written,
        out=out)
      map_files_written+=remainder_map_files_written
    else:
      map_files_written=[]
    return map_files_written
  
  def write_intermediate_maps(params,
      map_data=None,
      map_data_remaining=None,
      ncs_group_obj=None,
      tracking_data=None,
      out=sys.stdout):
  
    if map_data_remaining and params.output_files.remainder_map_file:
      write_ccp4_map(
         tracking_data.crystal_symmetry,params.output_files.remainder_map_file,
        map_data_remaining)
      print("Wrote output remainder map to %s" %(
         params.output_files.remainder_map_file), file=out)
  
    if params.segmentation.write_all_regions:
      for id in ncs_group_obj.selected_regions:
        region_mask=ncs_group_obj.edited_mask.deep_copy()
        s = (ncs_group_obj.edited_mask == -1)
        s |= (ncs_group_obj.edited_mask==id)
        region_mask = region_mask.set_selected(s,1)
        region_mask = region_mask.set_selected(~s,0)
  
        write_ccp4_map(tracking_data.crystal_symmetry,
            'mask_%d.ccp4' %id, region_mask)
        print("Wrote output mask for region %d to %s" %(id,
          "mask_%d.ccp4" %(id)), file=out)
  
  
  def iterate_search(params,
        map_data_remaining=None,
        map_data=None,
        ncs_obj=None,
        ncs_group_obj=None,
        scattered_points=None,
        tracking_data=None,
        out=sys.stdout):
  
  
    # Write out intermediate maps if desired
    if params.output_files.write_intermediate_maps:
      write_intermediate_maps(params,
        map_data=map_data,
        map_data_remaining=map_data_remaining,
        ncs_group_obj=ncs_group_obj,
        tracking_data=tracking_data,
        out=out)
    new_params=deepcopy(params)
    new_params.segmentation.iterate_with_remainder=False
    new_params.segmentation.density_threshold=None
    new_params.output_files.write_output_maps=False
    new_params.output_files.output_info_file=None
    if params.output_files.write_intermediate_maps:
      new_params.output_files.au_output_file_stem=\
        params.output_files.au_output_file_stem+"_cycle_2"
    else:
      new_params.output_files.au_output_file_stem=None
  
    fraction=params.segmentation.iteration_fraction
    if tracking_data.n_residues:
      new_n_residues=int(tracking_data.n_residues*fraction)
    new_solvent_fraction=max(0.001,min(0.999,
        1- (1-tracking_data.solvent_fraction)*fraction))
  
    new_tracking_data=deepcopy(tracking_data)
    if new_tracking_data.n_residues:
      new_tracking_data.set_n_residues(new_n_residues)
    new_tracking_data.set_solvent_fraction(new_solvent_fraction)
    new_tracking_data.set_origin_shift() # sets it to zero
    new_tracking_data.params.segmentation.starting_density_threshold=new_params.segmentation.starting_density_threshold # this is new
    print("\nIterating with remainder density", file=out)
    # NOTE: do not include pdb_hierarchy here unless you deep_copy it
    remainder_ncs_group_obj,dummy_remainder,remainder_tracking_data=run(
      None,params=new_params,
      map_data=map_data_remaining,
      ncs_obj=ncs_obj,
      target_scattered_points=scattered_points,
      tracking_data=new_tracking_data,
      is_iteration=True,
      out=out)
    if not remainder_ncs_group_obj: # Nothing to do
      return None
  
    # Combine the results to get remainder_id_dict
    #   remainder_id_dict[id_remainder]=id_nearby
  
    remainder_ncs_group_obj=combine_with_iteration(params,
       map_data=map_data,
       crystal_symmetry=tracking_data.crystal_symmetry,
       ncs_group_obj=ncs_group_obj,
       remainder_ncs_group_obj=remainder_ncs_group_obj,
       out=out)
  
    return remainder_ncs_group_obj
  
  def bounds_overlap(lower=None,upper=None,
            other_lower=None,other_upper=None,tol=1):
     for i in range(3):
       if       upper[i]+tol<other_lower[i]: return False
       if other_upper[i]+tol<lower[i]:       return False
     return True
  
  def combine_with_iteration(params,
      map_data=None,
      crystal_symmetry=None,
      ncs_group_obj=None,
      remainder_ncs_group_obj=None,
      out=sys.stdout):
  
    if not ncs_group_obj.selected_regions or not remainder_ncs_group_obj \
        or not remainder_ncs_group_obj.selected_regions:
      return None
  
    # see if any regions in ncs_obj overlap with remainder_ncs_group_obj...
    #   If so, combine
  
  
    remainder_id_dict={}
    for id_remainder in remainder_ncs_group_obj.selected_regions:
      best_id=None
      best_overlaps=None
      remainder_centers=\
          remainder_ncs_group_obj.region_scattered_points_dict[id_remainder]
      # figure out typical distance between scattered_points...
      touching_dist=get_touching_dist(remainder_centers)
  
      # Notice bounds of remainder region:
      r_lower,r_upper=get_bounds(
         ncs_group_obj=remainder_ncs_group_obj,id=id_remainder)
  
      for id in ncs_group_obj.selected_regions:
        # Skip if not likely to be very close...
        lower,upper=get_bounds(ncs_group_obj=ncs_group_obj,id=id)
        if not bounds_overlap(lower=lower,upper=upper,
            other_lower=r_lower,other_upper=r_upper):
          continue
  
        test_centers=ncs_group_obj.region_scattered_points_dict[id]
        dist=get_closest_dist(test_centers,remainder_centers)
        if touching_dist is not None and dist>touching_dist:
          continue
  
        bool_region_mask = ncs_group_obj.co.expand_mask(
          id_to_expand=ncs_group_obj.original_id_from_id[id],
          expand_size=params.segmentation.expand_size+1) # just touching
        s = (bool_region_mask== True)
        s &=  (remainder_ncs_group_obj.edited_mask==id_remainder)
        overlaps=s.count(True)
        if best_overlaps is None or overlaps>best_overlaps:
          best_overlaps=overlaps
          best_id=id
      if best_overlaps:
        print("\nCombining remainder id %d with original id %d (overlaps=%d)" %(
          id_remainder,best_id,best_overlaps), file=out)
        remainder_id_dict[id_remainder]=best_id
    remainder_ncs_group_obj.remainder_id_dict=remainder_id_dict
    return remainder_ncs_group_obj
  
  def get_touching_dist(centers,default=100.,min_dist=8.):
    mean_dist=0.
    mean_dist_n=0.
    nskip=max(1,len(centers)//10) # try to get 10
    for i in range(0,len(centers),nskip):
       if i==0:
         target=centers[1:]
       elif i==len(centers)-1:
         target=centers[:-1]
       else:
         target=centers[:i]
         target.extend(centers[i+1:])
       other=centers[i:i+1]
       if not target or not other: continue
       dist=get_closest_dist(target,other)
       if dist is not None:
         mean_dist+=dist
         mean_dist_n+=1.
    if mean_dist_n>0:
      return max(min_dist,2.0*mean_dist/mean_dist_n)
    else:
      return default
  
  def get_grid_units(map_data=None,crystal_symmetry=None,radius=None,
       out=sys.stdout):
      N_ = map_data.all()
      sx,sy,sz= 1/N_[0], 1/N_[1], 1/N_[2]
      sx_cart,sy_cart,sz_cart=crystal_symmetry.unit_cell().orthogonalize(
         [sx,sy,sz])
      grid_spacing=(sx_cart+sy_cart+sz_cart)/3.
      grid_units=int(radius/grid_spacing)
      min_cell_grid_units=min(N_[0], N_[1], N_[2])
      grid_units=max(1,min(grid_units,int(min_cell_grid_units/3)))
      print("Grid units representing %7.1f A will be %d" %(
         radius,grid_units), file=out)
      return grid_units
  
  def cut_out_map(map_data=None, crystal_symmetry=None,
      soft_mask=None,soft_mask_radius=None,resolution=None,
      shift_origin=None,
      min_point=None,max_point=None,out=sys.stdout):
    from cctbx import uctbx
    from cctbx import maptbx
    na = map_data.all() # tuple with dimensions
    for i in range(3):
      assert min_point[i] >= 0
      assert max_point[i] <= na[i]
    new_map_data = maptbx.copy(map_data, tuple(min_point), tuple(max_point))
    # NOTE: end point of map is max_point, so size of map (new all()) is
    #   (max_point-min_point+ (1,1,1))
    # shrink unit cell, angles are the same
    #  NOTE 2: the origin of output map will be min_point (not 0,0,0).
  
    shrunk_uc = []
    for i in range(3):
      shrunk_uc.append(
       crystal_symmetry.unit_cell().parameters()[i]*new_map_data.all()[i]/na[i] )
    uc_params=crystal_symmetry.unit_cell().parameters()
    new_unit_cell_box = uctbx.unit_cell(
      parameters=(shrunk_uc[0],shrunk_uc[1],shrunk_uc[2],
          uc_params[3],uc_params[4],uc_params[5]))
    new_crystal_symmetry=crystal.symmetry(
      unit_cell=new_unit_cell_box,space_group='p1')
  
    if soft_mask:
      if soft_mask_radius is None:
         soft_mask_radius=resolution
         assert soft_mask_radius is not None
  
      original_map_data=new_map_data.deep_copy()
      new_map_data,smoothed_mask_data=set_up_and_apply_soft_mask(
         map_data=new_map_data,
         shift_origin=shift_origin,
         crystal_symmetry=new_crystal_symmetry,
         resolution=resolution,
         radius=soft_mask_radius,out=out)
    else:
      original_map_data=None
      smoothed_mask_data=None
  
    return new_map_data, new_crystal_symmetry,\
      smoothed_mask_data,original_map_data
  
  def set_up_and_apply_soft_mask(map_data=None,shift_origin=None,
    crystal_symmetry=None,resolution=None,
    radius=None,out=sys.stdout):
  
      acc=map_data.accessor()
      map_data = map_data.shift_origin()
      new_acc=map_data.accessor()
  
      # Add soft boundary to mean around outside of mask
      # grid_units is how many grid units are about equal to soft_mask_radius
      grid_units=get_grid_units(map_data=map_data,
        crystal_symmetry=crystal_symmetry,radius=radius,out=out)
      grid_units=int(0.5+0.5*grid_units)
      from cctbx import maptbx
      zero_boundary_map=maptbx.zero_boundary_box_map(
         map_data,grid_units).result()
      # this map is zero's around the edge and 1 in the middle
      # multiply zero_boundary_map--smoothed & new_map_data and return
      print("Applying soft mask to boundary of cut out map", file=out)
      new_map_data,smoothed_mask_data=apply_soft_mask(map_data=map_data,
            mask_data=zero_boundary_map,
            rad_smooth=resolution,
            crystal_symmetry=crystal_symmetry,
            out=out)
      if new_acc != acc:
        new_map_data.reshape(acc)
        smoothed_mask_data.reshape(acc)
      return new_map_data,smoothed_mask_data
  
  def apply_shift_to_pdb_hierarchy(
      origin_shift=None,
      crystal_symmetry=None,
      pdb_hierarchy=None,out=sys.stdout):
  
    if origin_shift is not None:
      sites_cart=pdb_hierarchy.atoms().extract_xyz()
      sites_cart_shifted=sites_cart+\
        flex.vec3_double(sites_cart.size(), origin_shift)
      pdb_hierarchy.atoms().set_xyz(sites_cart_shifted)
  
    return pdb_hierarchy
  
  def apply_origin_shift(origin_shift=None,
      ncs_object=None,
      shifted_ncs_object=None,
      pdb_hierarchy=None,
      target_hierarchy=None,
      map_data=None,
      shifted_map_file=None,
      shifted_pdb_file=None,
      shifted_ncs_file=None,
      tracking_data=None,
      sharpening_target_pdb_inp=None,
      out=sys.stdout):
  
    if shifted_map_file:
        write_ccp4_map(tracking_data.crystal_symmetry,
        shifted_map_file,
        map_data)
        print("Wrote shifted map to %s" %(
          shifted_map_file), file=out)
        tracking_data.set_shifted_map_info(file_name=
          shifted_map_file,
          crystal_symmetry=tracking_data.crystal_symmetry,
          origin=map_data.origin(),
          all=map_data.all())
    if origin_shift: # Note origin shift does not change crystal_symmetry
  
      if pdb_hierarchy:
        pdb_hierarchy=apply_shift_to_pdb_hierarchy(
         origin_shift=origin_shift,
         crystal_symmetry=tracking_data.crystal_symmetry,
         pdb_hierarchy=pdb_hierarchy,
         out=out)
  
      if sharpening_target_pdb_inp:
        sharpening_target_pdb_inp=apply_shift_to_pdb_hierarchy(
         origin_shift=origin_shift,
         crystal_symmetry=tracking_data.crystal_symmetry,
         pdb_hierarchy=sharpening_target_pdb_inp.construct_hierarchy(),
         out=out).as_pdb_input()
  
      if target_hierarchy:
        target_hierarchy=apply_shift_to_pdb_hierarchy(
         origin_shift=origin_shift,
         crystal_symmetry=tracking_data.crystal_symmetry,
         pdb_hierarchy=target_hierarchy,
         out=out)
  
      from scitbx.math import  matrix
      if ncs_object and not shifted_ncs_object:
        shfted_ncs_object=ncs_object.coordinate_offset(
           coordinate_offset=matrix.col(origin_shift))
  
  
    if shifted_pdb_file and pdb_hierarchy:
        import iotbx.pdb
        f=open(shifted_pdb_file,'w')
        print(iotbx.pdb.format_cryst1_record(
           crystal_symmetry=tracking_data.crystal_symmetry), file=f)
        print(pdb_hierarchy.as_pdb_string(), file=f)
        f.close()
        print("Wrote shifted pdb file to %s" %(
          shifted_pdb_file), file=out)
        tracking_data.set_shifted_pdb_info(file_name=shifted_pdb_file,
        n_residues=pdb_hierarchy.overall_counts().n_residues)
  
  
    if shifted_ncs_file and shifted_ncs_object:
        shifted_ncs_object.format_all_for_group_specification(
           file_name=shifted_ncs_file)
        print("Wrote %s NCS operators for shifted map to %s" %(
           shifted_ncs_object.max_operators(),
           shifted_ncs_file), file=out)
        if tracking_data.input_ncs_info.has_updated_operators():
          print("NOTE: these may include additional operators added to fill the cell"+\
          " or\nhave fewer operators if not all applied.", file=out)
        tracking_data.set_shifted_ncs_info(file_name=shifted_ncs_file,
          number_of_operators=shifted_ncs_object.max_operators(),
          is_helical_symmetry=tracking_data.input_ncs_info.is_helical_symmetry)
        tracking_data.shifted_ncs_info.show_summary(out=out)
  
    return shifted_ncs_object,pdb_hierarchy,target_hierarchy,tracking_data,\
       sharpening_target_pdb_inp
  
  def restore_pdb(params,tracking_data=None,out=sys.stdout):
    if not params.output_files.restored_pdb:
      params.output_files.restored_pdb=\
         params.input_files.pdb_to_restore[:-4]+"_restored.pdb"
    print("Shifting origin of %s and writing to %s" %(
      params.input_files.pdb_to_restore,
      params.output_files.restored_pdb), file=out)
    os=tracking_data.origin_shift
    origin_shift=(-os[0],-os[1],-os[2])
    print("Origin shift will be: %.1f  %.1f  %.1f "%(origin_shift), file=out)
  
    import iotbx.pdb
    pdb_inp = iotbx.pdb.input(file_name=params.input_files.pdb_to_restore)
    pdb_hierarchy = pdb_inp.construct_hierarchy()
    pdb_hierarchy=apply_shift_to_pdb_hierarchy(
      origin_shift=origin_shift,
      crystal_symmetry=tracking_data.crystal_symmetry,
      pdb_hierarchy=pdb_hierarchy,
      out=out)
  
    f=open(params.output_files.restored_pdb,'w')
    print(iotbx.pdb.format_cryst1_record(
        crystal_symmetry=tracking_data.crystal_symmetry), file=f)
    print(pdb_hierarchy.as_pdb_string(), file=f)
    f.close()
    print("Wrote restored pdb file to %s" %(
       params.output_files.restored_pdb), file=out)
  
  def find_threshold_in_map(target_points=None,
        map_data=None,
        iter_max=10):
  
    map_1d=map_data.as_1d()
    map_mean=map_1d.min_max_mean().mean
    map_max=map_1d.min_max_mean().max
    map_min=map_1d.min_max_mean().min
  
    cutoff=map_mean
    low=map_min
    high=map_max
  
    for iter in range(iter_max):
      s = (map_1d >cutoff)
      n_cutoff=s.count(True)
      if n_cutoff == target_points:
        return cutoff
      elif n_cutoff < target_points: # lower it
        high=cutoff
        cutoff=0.5*(cutoff+low)
      else:  # raise it
        low=cutoff
        cutoff=0.5*(cutoff+high)
    return cutoff
  
  
  def remove_points(mask,remove_points=None):
    keep_points=(remove_points==False)
    new_mask=(mask & keep_points)
    return new_mask
  
  def get_ncs_sites_cart(sites_cart=None,ncs_id=None,
       ncs_obj=None, unit_cell=None, ncs_in_cell_only=True):
  
    ncs_sites_cart=flex.vec3_double()
    if not ncs_obj or not ncs_obj.ncs_groups() or not ncs_obj.ncs_groups()[0] or \
       not ncs_obj.ncs_groups()[0].translations_orth():
      return ncs_sites_cart
  
    # identify ncs-related points
    ncs_group=ncs_obj.ncs_groups()[0]
    identity_op=ncs_group.identity_op_id()
    ncs_sites_cart=flex.vec3_double()
    for xyz_cart in sites_cart:
      for i0 in range(len(ncs_group.translations_orth())):
        if i0==identity_op: continue
        if ncs_id is not None and i0!=ncs_id: continue
        r=ncs_group.rota_matrices_inv()[i0] # inverse maps pos 0 on to pos i
        t=ncs_group.translations_orth_inv()[i0]
        new_xyz_cart=r * matrix.col(xyz_cart) + t
        ncs_sites_cart.append(new_xyz_cart)
    if ncs_in_cell_only:
      new_sites_cart=flex.vec3_double()
      ncs_sites_frac=unit_cell.fractionalize(ncs_sites_cart)
      for site_frac,site_cart in zip(ncs_sites_frac,ncs_sites_cart):
        if site_frac[0]>=0 and site_frac[0]<=1 and  \
           site_frac[1]>=0 and site_frac[1]<=1 and  \
           site_frac[2]>=0 and site_frac[2]<=1:
          new_sites_cart.append(site_cart)
      ncs_sites_cart=new_sites_cart
  
    return ncs_sites_cart
  
  def get_ncs_mask(map_data=None,unit_cell=None,ncs_object=None,
     starting_mask=None,radius=None,expand_radius=None,overall_mask=None,
     every_nth_point=None):
  
    assert every_nth_point is not None
    if not expand_radius: expand_radius=2.*radius
  
    working_au_mask=starting_mask.deep_copy()
  
    working_ncs_mask=mask_from_sites_and_map(  # empty ncs mask
      map_data=map_data,unit_cell=unit_cell,
      sites_cart=flex.vec3_double(),radius=radius,overall_mask=overall_mask)
  
    au_points_last=working_au_mask.count(True)
    ncs_points_last=working_ncs_mask.count(True)
  
    max_tries=10000
    for ii in range(max_tries): # just a big number; should take just a few
  
      # Find all points in au (sample every_nth_point in grid)
  
      au_sites_cart=get_marked_points_cart(mask_data=working_au_mask,
       unit_cell=unit_cell,every_nth_point=every_nth_point,
       boundary_radius=radius)
  
      # Find all points ncs-related to marked point in mask
      ncs_sites_cart=get_ncs_sites_cart(sites_cart=au_sites_cart,
         ncs_obj=ncs_object,unit_cell=unit_cell,ncs_in_cell_only=True)
  
      # Expand au slightly with all points near to au_sites_cart
      new_au_mask=mask_from_sites_and_map(
        map_data=map_data,unit_cell=unit_cell,
        sites_cart=au_sites_cart,radius=radius,overall_mask=overall_mask)
      working_au_mask=(working_au_mask | new_au_mask) # add on to existing
      keep_points=(working_ncs_mask==False)  # cross off those in  ncs
      working_au_mask=(working_au_mask & keep_points)
  
      # mark ncs au with all points not in au that are close to ncs_sites_cart
      new_ncs_mask=mask_from_sites_and_map(
        map_data=map_data,unit_cell=unit_cell,
        sites_cart=ncs_sites_cart,radius=radius,overall_mask=overall_mask)
      keep_points=(working_au_mask==False)  # cross off those in au
      new_ncs_mask=(new_ncs_mask & keep_points)
      working_ncs_mask=(new_ncs_mask | working_ncs_mask) # add on to existing
  
      au_points=working_au_mask.count(True)
      ncs_points=working_ncs_mask.count(True)
      if au_points==au_points_last and ncs_points==ncs_points_last:
        break
      au_points_last=au_points
      ncs_points_last=ncs_points
      # Now expand the au and repeat
  
      working_au_mask=mask_from_sites_and_map(
        map_data=map_data,unit_cell=unit_cell,
        sites_cart=au_sites_cart,radius=expand_radius,overall_mask=overall_mask)
      keep_points=(working_ncs_mask==False)  # cross off those in  ncs
      working_au_mask=(working_au_mask & keep_points)
  
    return working_au_mask,working_ncs_mask
  
  def renormalize_map_data(
    map_data=None,solvent_fraction=None):
    sd=max(0.0001,map_data.sample_standard_deviation())
    if solvent_fraction >= 10.: solvent_fraction=solvent_fraction/100.
    assert solvent_fraction > 0 and solvent_fraction < 1
    scaled_sd=sd/(1-solvent_fraction)**0.5
    map_data=(map_data-map_data.as_1d().min_max_mean().mean)/scaled_sd
    return map_data
  
  
  def mask_from_sites_and_map(
      map_data=None,unit_cell=None,
      sites_cart=None,radius=None,overall_mask=None):
    assert radius is not None
    from cctbx import maptbx
  
    sel = maptbx.grid_indices_around_sites(
        unit_cell  = unit_cell,
        fft_n_real = map_data.focus(),
        fft_m_real = map_data.all(),
        sites_cart = sites_cart,
        site_radii = flex.double(sites_cart.size(), radius))
    map_data_1d=map_data.as_1d()
    mask=(map_data_1d==0 and map_data_1d==1)  # 1D bool array all False
    mask.set_selected(sel, True)  # mark points around sites
    mask.reshape(map_data.accessor())
    if overall_mask:
      assert overall_mask.all()==mask.all()
      mask=(mask & overall_mask)
    return mask
  
  def set_radius(unit_cell=None,map_data=None,every_nth_point=None):
    # Set radius so that radius will capture all points on grid if sampled
    #  on every_nth_point
    a,b,c = unit_cell.parameters()[:3]
    nx,ny,nz=map_data.all()
    # furthest possible minimum distance between grid points
    max_diagonal_between_sampled=every_nth_point*(
        (a/nx)**2+(b/ny)**2+(c/nz)**2)**0.5
    radius=max_diagonal_between_sampled*0.55  # big enough to cover everything
    return radius
  
  def get_marked_points_cart(mask_data=None,unit_cell=None,
     every_nth_point=3,boundary_radius=None):
    # return list of cartesian coordinates of grid points that are marked
    # only sample every every_nth_point in each direction...
    assert mask_data.origin() == (0,0,0)
    nx,ny,nz=mask_data.all()
    if boundary_radius:
      # How far from edges shall we stay:
      grid_frac=(1./nx,1./ny,1./nz)
      grid_orth=unit_cell.orthogonalize(grid_frac)
      boundary_grid_points=0
      for go in grid_orth:
        bgp=int(0.99+boundary_radius/go)
        boundary_grid_points=max(boundary_grid_points,bgp)
    else:
      boundary_grid_points=0
  
    marked_points=maptbx.marked_grid_points(
      map_data=mask_data,
      every_nth_point=every_nth_point).result()
    sites_frac=flex.vec3_double()
    boundary_points_skipped=0
    for grid_point in marked_points:
      if boundary_grid_points:
        if \
           grid_point[0]<boundary_grid_points or \
           grid_point[0]>nx-boundary_grid_points or \
           grid_point[1]<boundary_grid_points or \
           grid_point[1]>ny-boundary_grid_points or \
           grid_point[2]<boundary_grid_points or \
           grid_point[2]>nz-boundary_grid_points:  # XXX was typo previously
          boundary_points_skipped+=1
          continue
      sites_frac.append(
          (grid_point[0]/nx,
           grid_point[1]/ny,
           grid_point[2]/nz))
  
    sites_cart=unit_cell.orthogonalize(sites_frac)
    return sites_cart
  
  def get_overall_mask(
      map_data=None,
      mask_threshold=None,
      fraction_of_max_mask_threshold=None,
      mask_padding_fraction=None,
      solvent_fraction=None,
      crystal_symmetry=None,
      radius=None,
      resolution=None,
      d_max=100000.,
      out=sys.stdout):
  
    # Make a local SD map from our map-data
    from cctbx.maptbx import crystal_gridding
    from cctbx import sgtbx
    cg=crystal_gridding(
          unit_cell=crystal_symmetry.unit_cell(),
          space_group_info=sgtbx.space_group_info(number=1), # Always
          pre_determined_n_real=map_data.all())
  
    if not resolution:
      from cctbx.maptbx import d_min_from_map
      resolution=d_min_from_map(
        map_data,crystal_symmetry.unit_cell(), resolution_factor=1./4.)
      print("\nEstimated resolution of map: %6.1f A\n" %(
       resolution), file=out)
  
    if radius:
      smoothing_radius=2.*radius
    else:
      smoothing_radius=2.*resolution
  
    from mmtbx.command_line.map_to_structure_factors import run as map_to_sf
    args=['d_min=None','box=True']
    from libtbx.utils import null_out
    map_coeffs=map_to_sf(args=args,
           space_group_number=1, # always p1 cell for this
           ccp4_map=make_ccp4_map(map_data,crystal_symmetry.unit_cell()),
           return_as_miller_arrays=True,nohl=True,out=null_out())
    if not map_coeffs:
      raise Sorry("No map coeffs obtained")
    map_coeffs=map_coeffs.resolution_filter(d_min=resolution,d_max=d_max)
  
    complete_set = map_coeffs.complete_set()
    stol = flex.sqrt(complete_set.sin_theta_over_lambda_sq().data())
    import math
    w = 4 * stol * math.pi * smoothing_radius
    sphere_reciprocal = 3 * (flex.sin(w) - w * flex.cos(w))/flex.pow(w, 3)
  
    try:
      temp = complete_set.structure_factors_from_map(
        flex.pow2(map_data-map_data.as_1d().min_max_mean().mean))
    except Exception as e:
      print(e, file=out)
      raise Sorry("The sampling of the map appears to be too low for a "+
        "\nresolution of %s. Try using a larger value for resolution" %(
         resolution))
    fourier_coeff=complete_set.array(data=temp.data()*sphere_reciprocal)
    sd_map=fourier_coeff.fft_map(
        crystal_gridding=cg,
        ).apply_volume_scaling().real_map_unpadded()
    assert sd_map.all()==map_data.all()
    # now use sd_map
  
    # First mask out the map based on threshold
    mm=sd_map.as_1d().min_max_mean()
    max_in_sd_map=mm.max
    mean_in_map=mm.mean
    min_in_map=mm.min
    print("Highest value in SD map is %7.2f. Mean is %7.2f .  Lowest is %7.2f " %(
      max_in_sd_map,
      mean_in_map,
      min_in_map), file=out)
  
    if fraction_of_max_mask_threshold:
      mask_threshold=fraction_of_max_mask_threshold*max_in_sd_map
      print("Using fraction of max as threshold: %.3f " %(
          fraction_of_max_mask_threshold), \
          "which is threshold of %.3f" %(mask_threshold), file=out)
      if mask_padding_fraction:
        # Adjust threshold to increase by mask_padding_fraction, proportional
        #  to fraction available
        overall_mask=(sd_map>= mask_threshold)
        current_above_threshold=overall_mask.count(True)/overall_mask.size()
        # current+(1-current)*pad
        additional_padding=(1-current_above_threshold)*mask_padding_fraction
        target_above_threshold=min(
          0.99,current_above_threshold+additional_padding)
        print("Target with padding of %.2f will be %.2f" %(
           mask_padding_fraction,target_above_threshold), file=out)
        solvent_fraction=(1-target_above_threshold)
        mask_threshold=None
  
    if mask_threshold:
      print("Cutoff for mask will be input threshold", file=out)
      threshold=mask_threshold
    else:  # guess based on solvent_fraction
      if solvent_fraction is None:
         print("Guessing solvent fraction of 0.9", file=out)
         solvent_fraction=0.9 # just guess
      threshold=find_threshold_in_map(target_points=int(
        (1.-solvent_fraction)*sd_map.size()),
        map_data=sd_map)
      print("Cutoff will be threshold marking about %7.1f%% of cell" %(
        100.*(1.-solvent_fraction)), file=out)
  
    overall_mask=(sd_map>= threshold)
    print("Model region of map "+\
      "(density above %7.3f )" %( threshold) +" includes %7.1f%% of map" %(
        100.*overall_mask.count(True)/overall_mask.size()), file=out)
    return overall_mask,max_in_sd_map,sd_map
  
  def get_skew(data=None):
    mean=data.min_max_mean().mean
    sd=data.standard_deviation_of_the_sample()
    x=data-mean
    return (x**3).min_max_mean().mean/sd**3
  
  def get_kurtosis(data=None):
    mean=data.min_max_mean().mean
    sd=data.standard_deviation_of_the_sample()
    x=data-mean
    return (x**4).min_max_mean().mean/sd**4
  
  def score_map(map_data=None,
          sharpening_info_obj=None,
          solvent_fraction=None,
          fraction_occupied=None,
          wrapping=None,
          sa_percent=None,
          region_weight=None,
          max_regions_to_test=None,
          scale_region_weight=False,
          out=sys.stdout):
    if sharpening_info_obj:
      solvent_fraction=sharpening_info_obj.solvent_fraction
      wrapping=sharpening_info_obj.wrapping
      fraction_occupied=sharpening_info_obj.fraction_occupied
      sa_percent=sharpening_info_obj.sa_percent
      region_weight=sharpening_info_obj.region_weight
      max_regions_to_test=sharpening_info_obj.max_regions_to_test
    else:
      sharpening_info_obj=sharpening_info()
    if solvent_fraction is None:  # skip SA score
      sharpening_info_obj.adjusted_sa=0.
      assert sharpening_info_obj.sharpening_target=='kurtosis'
    else:  # usual
      map_data=renormalize_map_data(
         map_data=map_data,solvent_fraction=solvent_fraction)
  
      target_in_all_regions=map_data.size()*fraction_occupied*(1-solvent_fraction)
      print("\nTarget number of points in all regions: %.0f" %(
        target_in_all_regions), file=out)
  
      threshold=find_threshold_in_map(target_points=int(
           target_in_all_regions),map_data=map_data)
      print("Cutoff will be threshold of %7.2f marking %7.1f%% of cell" %(
                threshold,100.*(1.-solvent_fraction)), file=out)
  
      co,sorted_by_volume,min_b,max_b=get_co(
        map_data=map_data.deep_copy(),
         threshold=threshold,wrapping=wrapping)
  
      if len(sorted_by_volume)<2:
        return sharpening_info_obj# skip it, nothing to do
  
      target_sum= sa_percent* target_in_all_regions*0.01
      print("Points for %.1f percent of target in all regions: %.1f" %(
          sa_percent,target_sum), file=out)
  
      cntr=0
      sum_v=0.
      sum_new_v=0.
      for p in sorted_by_volume[1:max_regions_to_test+2]:
        cntr+=1
        v,i=p
        sum_v+=v
        bool_expanded=co.expand_mask(id_to_expand=i,expand_size=1)
        new_v=bool_expanded.count(True)-v
        sum_new_v+=new_v
        sa_ratio=new_v/v
        if sum_v>=target_sum: break
      sa_ratio=sum_new_v/max(1.,sum_v) # ratio of SA to volume
      regions=len(sorted_by_volume[1:])
      normalized_regions=regions/max(1,target_in_all_regions)
      skew=get_skew(map_data.as_1d())
  
      if scale_region_weight:
        solvent_fraction_std=0.85 # typical value, ends up as scale on weight
        region_weight_scale=(1.-solvent_fraction)/(1.-solvent_fraction_std)
        region_weight_use=region_weight*region_weight_scale
      else:
        region_weight_use=region_weight
      sharpening_info_obj.adjusted_sa=\
         sa_ratio - region_weight_use*normalized_regions
      sharpening_info_obj.sa_ratio=sa_ratio
      sharpening_info_obj.normalized_regions=normalized_regions
  
    sharpening_info_obj.kurtosis=get_kurtosis(map_data.as_1d())
    if sharpening_info_obj.sharpening_target=='kurtosis':
      sharpening_info_obj.score=sharpening_info_obj.kurtosis
    else:
      sharpening_info_obj.score=sharpening_info_obj.adjusted_sa
    return sharpening_info_obj
  
  def sharpen_map_with_si(sharpening_info_obj=None,
       f_array_normalized=None,
       f_array=None,phases=None,
       map_data=None,
       overall_b=None,
       resolution=None,
       out=sys.stdout):
    si=sharpening_info_obj
  
  
    if si.sharpening_method=='no_sharpening':
       return map_and_b_object(map_data=map_data)
  
    if map_data and (not f_array or not phases):
      map_coeffs,dummy=get_f_phases_from_map(map_data=map_data,
         crystal_symmetry=si.crystal_symmetry,
         d_min=si.resolution,
         d_min_ratio=si.d_min_ratio,
         return_as_map_coeffs=True,
         scale_max=si.scale_max,
         out=out)
      f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
  
    if si.remove_aniso:
      if si.use_local_aniso and \
        (si.local_aniso_in_local_sharpening or
         (si.local_aniso_in_local_sharpening is None and si.ncs_copies==1)) and \
           si.original_aniso_obj: # use original
        aniso_obj=si.original_aniso_obj
        print("\nRemoving aniso from map using saved aniso object before sharpening\n", file=out)
      else:
        print("\nRemoving aniso from map before sharpening\n", file=out)
        aniso_obj=None
      from cctbx.maptbx.refine_sharpening import analyze_aniso
      f_array,f_array_ra=analyze_aniso(
          aniso_obj=aniso_obj,
          remove_aniso=si.remove_aniso,
          f_array=f_array,resolution=si.resolution,out=out)
  
    if si.is_model_sharpening() or si.is_half_map_sharpening():
      from cctbx.maptbx.refine_sharpening import scale_amplitudes
      ff=f_array.phase_transfer(phase_source=phases,deg=True)
      map_and_b=scale_amplitudes(
        map_coeffs=f_array.phase_transfer(phase_source=phases,deg=True),
        si=si,overall_b=overall_b,out=out)
      return map_and_b
  
    elif si.is_resolution_dependent_sharpening():
      if f_array_normalized is None:
  
        from cctbx.maptbx.refine_sharpening import get_sharpened_map,\
         quasi_normalize_structure_factors
        (d_max,d_min)=f_array.d_max_min()
        if not f_array.binner():
          f_array.setup_binner(n_bins=si.n_bins,d_max=d_max,d_min=d_min)
        f_array_normalized=quasi_normalize_structure_factors(
            f_array,set_to_minimum=0.01)
      map_data=get_sharpened_map(ma=f_array_normalized,phases=phases,
         b=si.resolution_dependent_b,resolution=si.resolution,n_real=si.n_real,
         d_min_ratio=si.d_min_ratio)
      return map_and_b_object(map_data=map_data)
  
    else:
      map_and_b=apply_sharpening(n_real=si.n_real,
            f_array=f_array,phases=phases,
            sharpening_info_obj=si,
            crystal_symmetry=si.crystal_symmetry,
            out=null_out())
      return map_and_b
  
  def put_bounds_in_range(
       lower_bounds=None,upper_bounds=None,
       box_size=None,
       n_buffer=None,
       n_real=None,out=sys.stdout):
    # put lower and upper inside (0,n_real) and try to make size at least minimum
  
    new_lb=[]
    new_ub=[]
    print("Putting bounds in range...(%s,%s,%s) to (%s,%s,%s)" %(
         tuple(list(lower_bounds)+list(upper_bounds))), file=out)
    if n_buffer:
       print("Buffer of %s added" %(n_buffer), file=out)
    for lb,ub,ms,nr in zip(lower_bounds,upper_bounds,box_size,n_real):
      if ub<lb: ub=lb
      if lb >ub: lb=ub
      extra=(ms-(ub-lb))//2
      lb=lb-extra
      ub=ub+extra
  
      if n_buffer:
         lb=lb-n_buffer
         ub=ub+n_buffer
  
      if lb<0:
        shift=-lb
        lb+=shift
        ub+=shift
      boundary=int(ms-(ub-lb+1))//2
      if boundary>0:
         lb=lb-boundary
         ub=ub+boundary
      if lb<0: lb=0
      if ub>nr: ub=nr
      new_lb.append(lb)
      new_ub.append(ub)
    print("New bounds ...(%s,%s,%s) to (%s,%s,%s)" %(
         tuple(list(new_lb)+list(new_ub))), file=out)
    return tuple(new_lb),tuple(new_ub)
  
  def get_iterated_solvent_fraction(map=None,
      verbose=None,
      resolve_size=None,
      crystal_symmetry=None,
      mask_padding_fraction=None,
      fraction_of_max_mask_threshold=None,
      cell_cutoff_for_solvent_from_mask=None,
      mask_resolution=None,
      out=sys.stdout):
    if cell_cutoff_for_solvent_from_mask and \
     crystal_symmetry.unit_cell().volume() > cell_cutoff_for_solvent_from_mask**3:
       #go directly to low_res_mask
       return get_solvent_fraction_from_low_res_mask(
        crystal_symmetry=crystal_symmetry,
        map_data=map.deep_copy(),
        mask_padding_fraction=mask_padding_fraction,
        fraction_of_max_mask_threshold=fraction_of_max_mask_threshold,
        mask_resolution=mask_resolution)
  
    try:
      from phenix.autosol.map_to_model import iterated_solvent_fraction
      solvent_fraction=iterated_solvent_fraction(
        crystal_symmetry=crystal_symmetry,
        map_as_double=map,
        verbose=verbose,
        resolve_size=resolve_size,
        return_solvent_fraction=True,
        out=out)
      if solvent_fraction<=0.989:  # means that it was 0.99 which is hard limit
        return solvent_fraction
      else:  # use backup method
        return get_solvent_fraction_from_low_res_mask(
          crystal_symmetry=crystal_symmetry,
          map_data=map.deep_copy(),
          mask_padding_fraction=mask_padding_fraction,
          fraction_of_max_mask_threshold=fraction_of_max_mask_threshold,
          mask_resolution=mask_resolution)
    except Exception as e:
      # catch case where map was not on proper grid
      if str(e).find("sym equiv of a grid point must be a grid point")>-1:
        print("\nSpace group:%s \n Unit cell: %s \n Gridding: %s \nError message: %s" %(
          crystal_symmetry.space_group().info(),
          str(crystal_symmetry.unit_cell().parameters()),
          str(map.all()),str(e)), file=out)
        raise Sorry(
        "The input map seems to be on a grid incompatible with crystal symmetry"+
           "\n(symmetry equivalents of a grid point must be on "+
            "an integer grid point)")
      elif str(e).find("maximum size for resolve is")>-1:
        raise Sorry(str(e)+
         "\nIt may be possible to go on by supplying solvent content"+
        "or molecular_mass")
  
      # Try to get solvent fraction with low_res mask
      return get_solvent_fraction_from_low_res_mask(
        crystal_symmetry=crystal_symmetry,
        map_data=map.deep_copy(),
        mask_padding_fraction=mask_padding_fraction,
        fraction_of_max_mask_threshold=fraction_of_max_mask_threshold,
        mask_resolution=mask_resolution)
  
  def get_solvent_fraction_from_low_res_mask(
        crystal_symmetry=None,map_data=None,
        fraction_of_max_mask_threshold=None,
        mask_padding_fraction=None,
        mask_resolution=None,
        out=sys.stdout):
  
    overall_mask,max_in_sd_map,sd_map=get_overall_mask(map_data=map_data,
      fraction_of_max_mask_threshold=fraction_of_max_mask_threshold,
      mask_padding_fraction=mask_padding_fraction,
      crystal_symmetry=crystal_symmetry,
      resolution=mask_resolution,
      out=out)
  
    solvent_fraction=overall_mask.count(False)/overall_mask.size()
    print("Solvent fraction from overall mask: %.3f " %(solvent_fraction), file=out)
    return solvent_fraction
  
  
  def get_solvent_fraction_from_molecular_mass(
          crystal_symmetry=None,molecular_mass=None,out=sys.stdout):
       map_volume=crystal_symmetry.unit_cell().volume()
       density_factor=1000*1.23 # just protein density, close enough...
       mm=molecular_mass
       molecule_fraction= mm*density_factor/map_volume
       if molecule_fraction > 1 and mm > 1000: mm=mm/1000  # was in Da
  
       solvent_fraction=max(0.01,min(1.,1 - (
           mm*density_factor/map_volume)))
       print("Solvent content of %7.2f from molecular mass of %7.1f kDa" %(
       solvent_fraction,mm), file=out)
       return solvent_fraction
  
  
  def set_up_si(var_dict=None,crystal_symmetry=None,
        is_crystal=None,
        ncs_copies=None,n_residues=None,
        solvent_fraction=None,molecular_mass=None,pdb_inp=None,map=None,
        auto_sharpen=True,half_map_data_list=None,verbose=None,
        out=sys.stdout):
      si=sharpening_info(n_real=map.all())
      args=[]
      auto_sharpen_methods=var_dict.get('auto_sharpen_methods')
      if auto_sharpen_methods and auto_sharpen_methods != ['None'] and \
          len(auto_sharpen_methods)==1:
        sharpening_method=auto_sharpen_methods[0]
      else:
        sharpening_method=None
  
      for param in [
         'verbose','resolve_size','seq_file', 'sequence',
         'box_size',
         'target_n_overlap',
         'restrict_map_size',
         'box_center','remove_aniso',
         'input_weight_map_pickle_file', 'output_weight_map_pickle_file',
         'read_sharpened_maps', 'write_sharpened_maps', 'select_sharpened_map',
         'output_directory',
         'smoothing_radius','use_local_aniso',
         'local_aniso_in_local_sharpening',
         'overall_before_local',
         'local_sharpening',
         'box_in_auto_sharpen',
         'density_select_in_auto_sharpen',
         'density_select_threshold_in_auto_sharpen',
         'use_weak_density',
         'resolution',
         'd_min_ratio',
         'scale_max',
         'input_d_cut',
         'b_blur_hires',
         'discard_if_worse',
         'mask_atoms','mask_atoms_atom_radius','value_outside_atoms',
         'soft_mask',
         'tol_r','abs_tol_t',
         'rel_tol_t',
         'require_helical_or_point_group_symmetry',
         'max_helical_operators',
         'allow_box_if_b_iso_set',
         'max_box_fraction',
         'cc_cut',
         'max_cc_for_rescale',
         'scale_using_last',
         'density_select_max_box_fraction',
         'k_sharpen',
         'optimize_b_blur_hires',
         'iterate',
         'optimize_d_cut',
          'residual_target','sharpening_target',
         'search_b_min','search_b_max','search_b_n','adjust_region_weight',
         'region_weight_method',
          'region_weight_factor',
          'region_weight_buffer',
          'region_weight_default',
          'target_b_iso_ratio',
          'signal_min',
          'buffer_radius',
          'wang_radius',
          'pseudo_likelihood',
          'target_b_iso_model_scale',
         'b_iso','b_sharpen',
         'resolution_dependent_b',
         'normalize_amplitudes_in_resdep',
         'region_weight',
         'sa_percent',
         'n_bins',
         'eps',
         'max_regions_to_test',
         'regions_to_keep',
         'fraction_occupied',
         'rmsd',
         'rmsd_resolution_factor',
         'k_sol',
         'b_sol',
         'fraction_complete',
         'nproc',
         'multiprocessing',
         'queue_run_command',
         'verbose',
           ]:
       x=var_dict.get(param)
       if x is not None:
         if type(x)==type([1,2,3]):
           xx=[]
           for k in x:
             xx.append(str(k))
           args.append("%s=%s" %(param," ".join(xx)))
         else:
           args.append("%s=%s" %(param,x))
      local_params=get_params_from_args(args)
  
      # Set solvent content from molecular_mass if present
      if molecular_mass and not solvent_fraction:
         solvent_fraction=get_solvent_fraction_from_molecular_mass(
          crystal_symmetry=crystal_symmetry,molecular_mass=molecular_mass,
          out=out)
  
  
  
      if (local_params.input_files.seq_file or
         local_params.crystal_info.sequence) and \
          not local_params.crystal_info.solvent_content and \
          not solvent_fraction: # 2017-12-19
          solvent_fraction=get_solvent_fraction(local_params,
            crystal_symmetry=crystal_symmetry,
            ncs_copies=ncs_copies,out=out)
      si.update_with_params(params=local_params,
        crystal_symmetry=crystal_symmetry,
        is_crystal=is_crystal,
        solvent_fraction=solvent_fraction,
        ncs_copies=ncs_copies,
        n_residues=n_residues,
        auto_sharpen=auto_sharpen,
        sharpening_method=sharpening_method,
        pdb_inp=pdb_inp,
        half_map_data_list=half_map_data_list,
        )
      return si
  
  def bounds_to_frac(b,map_data):
    a=map_data.all()
    return b[0]/a[0],b[1]/a[1], b[2]/a[2]
  
  def bounds_to_cart(b,map_data,crystal_symmetry=None):
    bb=bounds_to_frac(b,map_data)
    a,b,c=crystal_symmetry.unit_cell().parameters()[:3]
  
    return a*bb[0],b*bb[1], c*bb[2]
  
  def select_inside_box(lower_bounds=None,upper_bounds=None,xrs=None,
       hierarchy=None):
    if not hierarchy or not xrs:
      return None
    selection = flex.bool(xrs.scatterers().size())
    for atom_group in hierarchy.atom_groups():
      for atom in atom_group.atoms():
        if atom.xyz[0]>=lower_bounds[0] and \
           atom.xyz[0]<=upper_bounds[0] and \
           atom.xyz[1]>=lower_bounds[1] and \
           atom.xyz[1]<=upper_bounds[1] and \
           atom.xyz[2]>=lower_bounds[2] and \
           atom.xyz[2]<=upper_bounds[2]:
          selection[atom.i_seq]=True
  
    asc1=hierarchy.atom_selection_cache()
    return hierarchy.select(selection)
  
  def make_empty_map(template_map=None,value=0.):
      # Create empty map original_map_in_box
      empty_map=flex.double(template_map.as_1d().as_double().size(),value)
      empty_map.reshape(flex.grid(template_map.all()))
      return empty_map
  
  def sum_box_data(starting_map=None,box_map=None,
        lower_bounds=None,upper_bounds=None):
      # sum box data into starting_map
  
  
      #Pull out current starting_map data
      starting_box_data= maptbx.copy(starting_map, tuple(lower_bounds), tuple(upper_bounds))
      assert starting_box_data.all()==box_map.all()
  
      # Add to box_map
      starting_box_data=starting_box_data.as_1d()
      box_map_data=box_map.as_1d()
      starting_box_data+=box_map_data
  
      # put back into shape
      starting_box_data.reshape(flex.grid(box_map.all()))
  
      maptbx.set_box(
          map_data_from = starting_box_data,
          map_data_to   = starting_map,
          start         = lower_bounds,
          end           = upper_bounds)
      return starting_map
  
  
  def copy_box_data(starting_map=None,box_map=None,
        lower_bounds=None,upper_bounds=None):
      # Copy box data into original_map_in_box
      maptbx.set_box(
          map_data_from = box_map,
          map_data_to   = starting_map,
          start         = lower_bounds,
          end           = upper_bounds)
      return starting_map
  
  def select_box_map_data(si=None,
             map_data=None,
             first_half_map_data=None,
             second_half_map_data=None,
             pdb_inp=None,
             get_solvent_fraction=True,# XXX test not doing this...
             n_min=30, # at least 30 atoms to run model sharpening
             restrict_map_size=None,
             out=sys.stdout,local_out=sys.stdout):
  
    box_solvent_fraction=None
    solvent_fraction=si.solvent_fraction
    crystal_symmetry=si.crystal_symmetry
    box_size=si.box_size
    lower_bounds=None
    upper_bounds=None
    smoothed_box_mask_data=None
    original_box_map_data=None #
    n_buffer=None
  
    if  (not pdb_inp and not si.box_in_auto_sharpen) and (
        first_half_map_data and second_half_map_data):
        print("Creating density-based soft mask and applying to half-map data", file=out)
  
        if not si.soft_mask:
          raise Sorry(
           "Need to set soft_mask=True for half-map sharpening without model")
        # NOTE: could precede this by density_select on map_data, save bounds and
        #   cut out half-maps with those bounds. That case could
        #   cover si.soft_mask=False
  
        half_map_data_list=[first_half_map_data,second_half_map_data]
        box_mask_data,box_map,half_map_data_list,\
         box_solvent_fraction,smoothed_box_mask_data,original_box_map_data=\
         get_and_apply_soft_mask_to_maps(
          resolution=si.resolution,
          wang_radius=si.wang_radius,
          buffer_radius=si.buffer_radius,
          map_data=map_data,crystal_symmetry=crystal_symmetry,
          half_map_data_list=half_map_data_list,
          out=out)
  
        box_first_half_map,box_second_half_map=half_map_data_list
        box_crystal_symmetry=crystal_symmetry
        box_pdb_inp=pdb_inp
  
    elif pdb_inp or (
       si.density_select_in_auto_sharpen and not si.box_in_auto_sharpen):
  
    #   use map_box for pdb_inp (mask with model)
    #   also use map_box for density_select_in_auto_sharpen sharpening because
    #     need to use the same density select for all 3 maps.
  
    # XXX Perhaps we canuse above method for pdb_inp
  
      assert not si.local_sharpening
  
      if pdb_inp:
        print("Using map_box based on input model", file=out)
        hierarchy=pdb_inp.construct_hierarchy()
        max_box_fraction=si.max_box_fraction
        si.density_select_in_auto_sharpen=False
      else:
        #print >>out,"Using density_select in map_box"
        hierarchy=None
        assert si.density_select_in_auto_sharpen
        max_box_fraction=si.density_select_max_box_fraction
  
      #----------------------trimming model-------------------------------
      if si.box_center:  # Have model but center at box_center and trim hierarchy
        lower_bounds,upper_bounds=box_from_center(si=si,
          map_data=map_data,out=out)
        if si.soft_mask:
          n_buffer=get_grid_units(map_data=map_data,
            crystal_symmetry=crystal_symmetry,
            radius=si.resolution,out=out)
          n_buffer=int(0.5+n_buffer*1.5)
        else:
          n_buffer=0
        lower_bounds,upper_bounds=put_bounds_in_range(
         lower_bounds=lower_bounds,upper_bounds=upper_bounds,
         box_size=box_size,n_buffer=n_buffer,
         n_real=map_data.all(),out=out)
        lower_frac=bounds_to_frac(lower_bounds,map_data)
        upper_frac=bounds_to_frac(upper_bounds,map_data)
        lower_cart=bounds_to_cart(
           lower_bounds,map_data,crystal_symmetry=crystal_symmetry)
        upper_cart=bounds_to_cart(
          upper_bounds,map_data,crystal_symmetry=crystal_symmetry)
  
        if hierarchy: # trimming hierarchy to box and then using trimmed
                      # hierarchy in map_box to create actual box
          xrs=hierarchy.extract_xray_structure(
            crystal_symmetry=si.crystal_symmetry)
          # find everything in box
          sel_hierarchy=select_inside_box(lower_bounds=lower_cart,
           upper_bounds=upper_cart, xrs=xrs,hierarchy=hierarchy)
          n=sel_hierarchy.overall_counts().n_atoms
          print("Selected atoms inside box: %d" %(n), file=out)
          if n<n_min:
            print("Skipping...using entire structure", file=out)
          else:
            hierarchy=sel_hierarchy
      #----------------------end trimming model-------------------------------
  
  
      from mmtbx.command_line.map_box import run as run_map_box
      args=["keep_input_unit_cell_and_grid=False"]
      if si.density_select_in_auto_sharpen:
        args.append('density_select=True')
        #print >>out,"Using density_select in map_box"
        if si.density_select_threshold_in_auto_sharpen is not None:
          args.append('density_select_threshold=%s' %(
            si.density_select_threshold_in_auto_sharpen))
      elif si.box_in_auto_sharpen and not si.mask_atoms:
        print("Using map_box with model", file=out)
      elif si.mask_atoms:
        print("Using map_box with model and mask_atoms", file=out)
        args.append('mask_atoms=True')
        if si.mask_atoms_atom_radius:
          args.append('mask_atoms_atom_radius=%s' %(si.mask_atoms_atom_radius))
        if si.value_outside_atoms:
          args.append('value_outside_atoms=%s' %(si.value_outside_atoms))
        if si.soft_mask:
          print("Using soft mask", file=out)
          args.append('soft_mask=%s' %(si.soft_mask))
          args.append('soft_mask_radius=%s' %(si.resolution))
      else:
        raise Sorry("Unknown choice in select_box_data")
  
      if restrict_map_size:
        args.append('restrict_map_size=True')
      print("Getting map as box now", file=out)
      local_hierarchy=None
      if hierarchy:
         local_hierarchy=hierarchy.deep_copy() # run_map_box modifies its argument
      box=run_map_box(args,
          map_data=map_data,pdb_hierarchy=local_hierarchy,
         write_output_files=False,
         crystal_symmetry=crystal_symmetry,log=out)
  
      lower_bounds=box.gridding_first
      upper_bounds=box.gridding_last
  
      box_map=box.map_box
      box_map=scale_map(box_map,out=out)
      box_crystal_symmetry=box.box_crystal_symmetry
      box_pdb_inp=box.hierarchy.as_pdb_input()
      if first_half_map_data:
        print("Getting first map as box", file=out)
        if hierarchy:
          local_hierarchy=hierarchy.deep_copy() # required
        box_first=run_map_box(args,
          map_data=first_half_map_data,pdb_hierarchy=local_hierarchy,
         write_output_files=False,
         lower_bounds=lower_bounds,
         upper_bounds=upper_bounds,
         crystal_symmetry=crystal_symmetry,
         log=out)
        box_first_half_map=box_first.map_box.as_double()
      else:
        box_first_half_map=None
  
      if second_half_map_data:
        print("Getting second map as box", file=out)
        if hierarchy:
          local_hierarchy=hierarchy.deep_copy() # required
        box_second=run_map_box(args,
          map_data=second_half_map_data,pdb_hierarchy=local_hierarchy,
         write_output_files=False,
         lower_bounds=lower_bounds,
         upper_bounds=upper_bounds,
         crystal_symmetry=crystal_symmetry,
         log=out)
        box_second_half_map=box_second.map_box.as_double()
      else:
        box_second_half_map=None
  
    else: # cut out box based on box_center or regions
  
      if si.box_center:  # center at box_center
        print("Cutting out box based on center at (%.2f,%.2f,%.2f) " %( si.box_center), file=out)
        lower_bounds,upper_bounds=box_from_center(si=si,
          map_data=map_data,out=out)
      elif si.use_weak_density:
        print("Cutting out box based on centering on weak density", file=out)
        lower_bounds,upper_bounds=box_of_smallest_region(si=si,
             map_data=map_data,
             out=out)
      else:
        print("Cutting out box based on centering on strong density", file=out)
        lower_bounds,upper_bounds=box_of_biggest_region(si=si,
             map_data=map_data,
             out=out)
      if si.soft_mask:
        n_buffer=get_grid_units(map_data=map_data,
          crystal_symmetry=crystal_symmetry,
          radius=si.resolution,out=out)
        n_buffer=int(0.5+n_buffer*1.5)
      else:
        n_buffer=0
      lower_bounds,upper_bounds=put_bounds_in_range(
       lower_bounds=lower_bounds,upper_bounds=upper_bounds,
       box_size=box_size,n_buffer=n_buffer,
       n_real=map_data.all(),out=out)
  
      # select map data inside this box
      print("\nSelecting map data inside box", file=out)
  
      box_map,box_crystal_symmetry,\
           smoothed_box_mask_data,original_box_map_data=cut_out_map(
         map_data=map_data.as_double(),
         crystal_symmetry=crystal_symmetry,
         soft_mask=si.soft_mask,
         soft_mask_radius=si.resolution,
         resolution=si.resolution,
         shift_origin=True,
         min_point=lower_bounds, max_point=upper_bounds,out=out)
      box_pdb_inp=None
  
      if first_half_map_data:
        box_first_half_map,box_first_crystal_symmetry,\
          dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
         map_data=first_half_map_data.as_double(),
         crystal_symmetry=crystal_symmetry,
         soft_mask=si.soft_mask,
         soft_mask_radius=si.resolution,
         resolution=si.resolution,
         shift_origin=True,
         min_point=lower_bounds, max_point=upper_bounds,out=local_out)
      else:
        box_first_half_map=None
  
      if second_half_map_data:
        box_second_half_map,box_second_crystal_symmetry,\
           dummy_smoothed_box_mask_data,dummy_original_box_map_data=cut_out_map(
         map_data=second_half_map_data.as_double(),
         crystal_symmetry=crystal_symmetry,
         soft_mask=si.soft_mask,
         soft_mask_radius=si.resolution,
         resolution=si.resolution,
         shift_origin=True,
         min_point=lower_bounds, max_point=upper_bounds,out=local_out)
      else:
        box_second_half_map=None
  
    if not box_map or (
         (not pdb_inp and not second_half_map_data) and \
        box_map.size() > si.max_box_fraction* map_data.size()):
  
      return None,map_data,first_half_map_data,\
          second_half_map_data,crystal_symmetry,None,\
          smoothed_box_mask_data,None,None # no point
  
    # figure out solvent fraction in this box...
  
    if get_solvent_fraction: #
      if box_solvent_fraction is None:
        box_solvent_fraction=get_iterated_solvent_fraction(
          crystal_symmetry=box_crystal_symmetry,
          map=box_map,
          fraction_of_max_mask_threshold=si.fraction_of_max_mask_threshold,
          mask_resolution=si.resolution,
          out=out)
      if box_solvent_fraction is None:
        box_solvent_fraction=si.solvent_fraction
        print("Using overall solvent fraction for box", file=out)
      print("Local solvent fraction: %7.2f" %(box_solvent_fraction), file=out)
    else:
      box_solvent_fraction=None
  
    box_sharpening_info_obj=box_sharpening_info(
      lower_bounds=lower_bounds,
      upper_bounds=upper_bounds,
      n_real=box_map.all(),
      scale_max=si.scale_max,
      wrapping=False,
      crystal_symmetry=box_crystal_symmetry,
      solvent_fraction=box_solvent_fraction)
    return box_pdb_inp,box_map,box_first_half_map,box_second_half_map,\
        box_crystal_symmetry,box_sharpening_info_obj,\
        smoothed_box_mask_data,original_box_map_data,n_buffer
  
  def inside_zero_one(xyz):
    from scitbx.array_family import flex
    from scitbx.matrix import col
    offset=xyz-col((0.5,0.5,0.5))
    lower_int=offset.iround().as_vec3_double()
    return xyz-lower_int
  
  def move_xyz_inside_cell(xyz_cart=None,crystal_symmetry=None):
    xyz_local=flex.vec3_double()
    if type(xyz_cart)==type(xyz_local):
      xyz_local=xyz_cart
      is_single=False
    else:
      is_single=True
      xyz_local.append(xyz_cart)
  
    xyz_frac=crystal_symmetry.unit_cell().fractionalize(xyz_local)
    new_xyz_frac=inside_zero_one(xyz_frac)
    new_xyz_cart=crystal_symmetry.unit_cell().orthogonalize(new_xyz_frac)
    if is_single:
      return new_xyz_cart[0]
    else:
      return new_xyz_cart
  
  def box_from_center( si=None,
             map_data=None,
             out=sys.stdout):
      cx,cy,cz=si.crystal_symmetry.unit_cell().fractionalize(si.box_center)
      if cx<0 or cx>1 or cy<0 or cy>1 or cz<0 or cz>1:
         print("Moving box center inside (0,1)", file=out)
         si.box_center=move_xyz_inside_cell(
             xyz_cart=si.box_center,crystal_symmetry=si.crystal_symmetry)
      cx,cy,cz=si.crystal_symmetry.unit_cell().fractionalize(si.box_center)
      print("\nBox centered at (%7.2f,%7.2f,%7.2f) A" %(
        tuple(si.box_center)), file=out)
  
      ax,ay,az=map_data.all()
      cgx,cgy,cgz=int(0.5+ax*cx),int(0.5+ay*cy),int(0.5+az*cz),
      print("Box grid centered at (%d,%d,%d)\n" %(cgx,cgy,cgz), file=out)
      return (cgx,cgy,cgz),(cgx,cgy,cgz)
  
  def box_of_smallest_region(si=None,
             map_data=None,
             return_as_list=None,
             out=sys.stdout):
    return box_of_biggest_region(si=si,
             map_data=map_data,
             return_as_list=return_as_list,
             use_smallest=True,
             out=out)
  
  def box_of_biggest_region(si=None,
             map_data=None,
             return_as_list=None,
             use_smallest=False,
             out=sys.stdout):
      n_residues=si.n_residues
      ncs_copies=si.ncs_copies
      solvent_fraction=si.solvent_fraction
  
      b_vs_region=b_vs_region_info()
      co,sorted_by_volume,min_b,max_b,unique_expected_regions,best_score,\
         new_threshold,starting_density_threshold=\
          get_connectivity(
             b_vs_region=b_vs_region,
             map_data=map_data,
             n_residues=n_residues,
             ncs_copies=ncs_copies,
             solvent_fraction=solvent_fraction,
             min_volume=si.min_volume,
             min_ratio=si.min_ratio,
             fraction_occupied=si.fraction_occupied,
             wrapping=si.wrapping,
             residues_per_region=si.residues_per_region,
             max_ratio_to_target=si.max_ratio_to_target,
             min_ratio_to_target=si.min_ratio_to_target,
             min_ratio_of_ncs_copy_to_first=si.min_ratio_of_ncs_copy_to_first,
             starting_density_threshold=si.starting_density_threshold,
             density_threshold=si.density_threshold,
             crystal_symmetry=si.crystal_symmetry,
             chain_type=si.chain_type,
             verbose=si.verbose,
             out=out)
  
      if len(sorted_by_volume)<2:
        return # nothing to do
  
      if use_smallest:
        small_ratio=0.25
        maximum_position_ratio=0.75
        v1,i1=sorted_by_volume[1]
        v_small=small_ratio*v1
        maximum_position_small=maximum_position_ratio*(len(sorted_by_volume)-1)+1
  
        best_pos=1
        ii=0
        for v,i in sorted_by_volume[1:]:
          ii+=1
          if v < v_small: continue
          if ii > maximum_position_small: continue
          best_pos=ii
  
        v,i=sorted_by_volume[best_pos]
        print("\nVolume of target region %d: %d grid points: "%(best_pos,v), file=out)
      else: # usual
        v,i=sorted_by_volume[1]
        print("\nVolume of largest region: %d grid points: "%(v), file=out)
  
      print("Region %3d (%3d)  volume:%5d  X:%6d - %6d   Y:%6d - %6d  Z:%6d - %6d "%(
       1,i,v,
       min_b[i][0],max_b[i][0],
       min_b[i][1],max_b[i][1],
       min_b[i][2],max_b[i][2]), file=out)
  
      if (not return_as_list):
        return min_b[i],max_b[i]
  
      else: # return a list of centers
        centers_frac=flex.vec3_double()
        a1,a2,a3=map_data.all()
  
        for v,i in sorted_by_volume[1:]:
          centers_frac.append(
            tuple((
            (min_b[i][0]+max_b[i][0])/(2.*a1),
            (min_b[i][1]+max_b[i][1])/(2.*a2),
            (min_b[i][2]+max_b[i][2])/(2.*a3),
                 ))
                        )
        return centers_frac
  
  def get_fft_map(n_real=None,map_coeffs=None):
      from cctbx import maptbx
      from cctbx.maptbx import crystal_gridding
      if n_real:
        cg=crystal_gridding(
          unit_cell=map_coeffs.crystal_symmetry().unit_cell(),
          space_group_info=map_coeffs.crystal_symmetry().space_group_info(),
          pre_determined_n_real=n_real)
      else:
        cg=None
      ccs=map_coeffs.crystal_symmetry()
      fft_map = map_coeffs.fft_map( resolution_factor = 0.25,
         crystal_gridding=cg,
         symmetry_flags=maptbx.use_space_group_symmetry)
      fft_map.apply_sigma_scaling()
      return fft_map
  
  def average_from_bounds(lower,upper,grid_all=None):
    avg=[]
    for u,l in zip(upper,lower):
      avg.append(0.5*(u+l))
    if grid_all:
       avg_fract=[]
       for a,g in zip(avg,grid_all):
         avg_fract.append(a/g)
       avg=avg_fract
    return avg
  
  def get_ncs_copies(site_cart,ncs_object=None,
     only_inside_box=None,unit_cell=None):
  
  
    ncs_group=ncs_object.ncs_groups()[0]
    from scitbx.array_family import flex
    from scitbx.matrix import col
    sites_cart_ncs=flex.vec3_double()
  
    for t,r in zip(ncs_group.translations_orth_inv(),
                   ncs_group.rota_matrices_inv()):
  
      sites_cart_ncs.append(r * col(site_cart)  + t)
  
    if only_inside_box:
      assert unit_cell is not None
      sites_frac_ncs=unit_cell.fractionalize(sites_cart_ncs)
      new_sites_frac=flex.vec3_double()
      for x in sites_frac_ncs:
        if  x[0]>=0 and x[0]<=1  and \
            x[1]>=0 and x[1]<=1  and \
            x[2]>=0 and x[2]<=1:
          new_sites_frac.append(x)
      sites_cart_ncs=unit_cell.orthogonalize(new_sites_frac)
    return sites_cart_ncs
  
  
  def fit_bounds_inside_box(lower,upper,box_size=None,all=None):
    # adjust bounds so upper>lower and box size is at least box_size
    new_lower=[]
    new_upper=[]
    if box_size:
     for u,l,s,a in zip(upper,lower,box_size,all):
       ss=u-l+1
       delta=int((1+s-ss)/2) # desired increase in size, to subtract from l
       l=max(0,l-delta)
       ss=u-l+1
       delta=(s-ss) # desired increase in size, to add to u
       u=min(a-1,u+delta)
       l=max(0,l)
       new_lower.append(l)
       new_upper.append(u)
    else:
     for u,l,a in zip(upper,lower,all):
       u=min(a-1,u)
       l=max(0,l)
       new_lower.append(l)
       new_upper.append(u)
  
    return new_lower,new_upper
  
  def split_boxes(lower=None,upper=None,target_size=None,target_n_overlap=None,
       fix_target_size_and_overlap=None):
    # purpose: split the region from lower to upper into overlapping
    # boxes of size about target_size
    # NOTE: does not actually use target_n_overlap unless
    #     fix_target_size_and_overlap is set
  
    all_box_list=[]
    for l,u,ts in zip (lower,upper,target_size):
      n=u+1-l
      # offset defined by: ts-offset + ts-offset+...+ts =n
      #                 ts*n_box-offset*(n_box-1)=n approx
      #             n_box (ts-offset) +offset=n
      #             n_box= (n-offset)/(ts-offset)
      assert ts>target_n_overlap
      if fix_target_size_and_overlap:
        n_box=(n-target_n_overlap-1)/(ts-target_n_overlap)
        if n_box>int(n_box):
          n_box=int(n_box)+1
        else:
          n_box=int(n_box)
  
        new_target_size=ts
        offset=ts-target_n_overlap
      else: # original version
        n_box=max(1,(n+(3*ts//4))//ts)
        new_target_size=int(0.9+n/n_box)
        offset=int(0.9+(n-new_target_size)/max(1,n_box-1))
      box_list=[]
      for i in range(n_box):
        start_pos=max(l,l+i*offset)
        end_pos=min(u,start_pos+new_target_size)
        if fix_target_size_and_overlap:
          start_pos=max(0,min(start_pos,end_pos-new_target_size))
        box_list.append([start_pos,end_pos])
      all_box_list.append(box_list)
    new_lower_upper_list=[]
    for xs,xe in all_box_list[0]:
      for ys,ye in all_box_list[1]:
        for zs,ze in all_box_list[2]:
          new_lower_upper_list.append([(xs,ys,zs,),(xe,ye,ze,)])
    return new_lower_upper_list
  
  def get_target_boxes(si=None,ncs_obj=None,map=None,
      pdb_inp=None,out=sys.stdout):
  
    print(80*"-", file=out)
    print("Getting segmented map to ID locations for sharpening", file=out)
    print(80*"-", file=out)
  
    if si.input_weight_map_pickle_file:
      from libtbx import easy_pickle
      file_name=si.input_weight_map_pickle_file
      print("Loading segmentation data from %s" %(file_name), file=out)
      tracking_data=easy_pickle.load(file_name)
  
    else:
      args=[
          'resolution=%s' %(si.resolution),
          'seq_file=%s' %(si.seq_file),
          'sequence=%s' %(si.sequence),
          'solvent_content=%s' %(si.solvent_fraction),
          'auto_sharpen=False', # XXX could sharpen overall
          'write_output_maps=True',
          'add_neighbors=False',
          'density_select=False', ]
      if si.is_crystal:
        args.append("is_crystal=True")
      ncs_group_obj,remainder_ncs_group_obj,tracking_data=run(
       args,
       map_data=map.deep_copy(),
       ncs_obj=ncs_obj,
       crystal_symmetry=si.crystal_symmetry)
  
    if si.output_weight_map_pickle_file:
      from libtbx import easy_pickle
      file_name=os.path.join(si.output_directory,si.output_weight_map_pickle_file)
      print("Dumping segmentation data to %s" %(file_name), file=out)
      easy_pickle.dump(file_name,tracking_data)
  
    if not ncs_obj or ncs_obj.max_operators()==0:
      from mmtbx.ncs.ncs import ncs
      ncs_obj=ncs()
      ncs_obj.set_unit_ncs()
  
    print("Regions in this map:", file=out)
    centers_frac=flex.vec3_double()
    upper_bounds_list=[]
    lower_bounds_list=[]
    if pdb_inp and pdb_inp.atoms().extract_xyz().size()>1:
      xyz_list=pdb_inp.atoms().extract_xyz()
      i_end=xyz_list.size()
      n_centers=min(i_end,max(1,len(tracking_data.output_region_map_info_list)))
      n_steps=min(n_centers,xyz_list.size())
      i_step=int(0.5+min(i_end/2,i_end/n_steps)) # about n_centers but up to n_atoms
      i_start=max(1,int(0.5+i_step/2))
      from scitbx.matrix import col
      ma=map.all()
      for i in range(i_start,i_end,i_step):
        lower_cart=col(xyz_list[i])
        lower_frac=si.crystal_symmetry.unit_cell().fractionalize(lower_cart)
        lower=[
          int(0.5+ma[0]*lower_frac[0]),
          int(0.5+ma[1]*lower_frac[1]),
          int(0.5+ma[2]*lower_frac[2])]
        lower,upper=fit_bounds_inside_box(
          lower,lower,box_size=si.box_size,all=map.all())
        upper_bounds_list.append(upper)
        lower_bounds_list.append(lower)
        average_fract=average_from_bounds(lower,upper,grid_all=map.all())
        centers_frac.append(average_fract)
    else:
      for map_info_obj in tracking_data.output_region_map_info_list:
        lower,upper=map_info_obj.lower_upper_bounds()
        lower,upper=fit_bounds_inside_box(
          lower,upper,
          box_size=None, # take the whole region, not just center
          all=map.all())
        for lower,upper in split_boxes(lower=lower,upper=upper,
           target_size=si.box_size,
           target_n_overlap=si.target_n_overlap):
           upper_bounds_list.append(upper)
           lower_bounds_list.append(lower)
           average_fract=average_from_bounds(lower,upper,grid_all=map.all())
           centers_frac.append(average_fract)
  
  
    centers_cart=si.crystal_symmetry.unit_cell().orthogonalize(centers_frac)
  
  
    #  Make ncs-related centers
    print("NCS ops:",ncs_obj.max_operators(), file=out)
    centers_cart_ncs_list=[]
    for i in range(centers_cart.size()):
      centers_cart_ncs_list.append(get_ncs_copies(
         centers_cart[i],ncs_object=ncs_obj,only_inside_box=True,
         unit_cell=si.crystal_symmetry.unit_cell()) )
  
    all_cart=flex.vec3_double()
    for center_list in centers_cart_ncs_list:
      all_cart.extend(center_list)
  
    sharpening_centers_file=os.path.join(
        si.output_directory,"sharpening_centers.pdb")
    write_atoms(file_name=sharpening_centers_file,
      crystal_symmetry=si.crystal_symmetry,sites=centers_cart)
    ncs_sharpening_centers_file=os.path.join(
        si.output_directory,"ncs_sharpening_centers.pdb")
    write_atoms(file_name=ncs_sharpening_centers_file,
      crystal_symmetry=si.crystal_symmetry,sites=all_cart)
  
    print("\nSharpening centers (matching shifted_map_file).\n\n "+\
        "Written to: \n%s \n%s\n"%(
        sharpening_centers_file,ncs_sharpening_centers_file), file=out)
  
    for i in range(centers_cart.size()):
      print("Center: %s (%7.2f,%7.2f,%7.2f)  Bounds: %s :: %s " %(
          i,centers_cart[i][0],centers_cart[i][1],centers_cart[i][2],
            str(lower_bounds_list[i]),str(upper_bounds_list[i])), file=out)
  
    print(80*"-", file=out)
    print("Done getting segmented map to ID locations for sharpening", file=out)
    print(80*"-", file=out)
  
  
    return upper_bounds_list,lower_bounds_list,\
       centers_cart_ncs_list,centers_cart,all_cart
  
  def get_box_size(lower_bound=None,upper_bound=None):
    box_size=[]
    for lb,ub in zip(lower_bound,upper_bound):
      box_size.append(ub-lb)
    return box_size
  
  def mean_dist_to_nearest_neighbor(all_cart):
    if all_cart.size()<2:  # nothing to check
      return None
    sum_dist=0.
    sum_n=0.
    for i in range(all_cart.size()):
      xyz=all_cart[i:i+1]
      others=all_cart[:i]
      others.extend(all_cart[i+1:])
      sum_dist+=get_closest_dist(xyz,others)
      sum_n+=1.
    return sum_dist/max(1.,sum_n)
  
  
  def run_local_sharpening(si=None,
      auto_sharpen_methods=None,
      map=None,
      ncs_obj=None,
      half_map_data_list=None,
      pdb_inp=None,
      out=sys.stdout):
    print(80*"-", file=out)
    print("Running local sharpening", file=out)
    print(80*"-", file=out)
  
    # run auto_sharpen_map_or_map_coeffs with box_in_auto_sharpen=True and
    #   centered at different places.  Identify the places as centers of regions.
    #   Run on au of NCS and apply NCS to get remaining positions
  
    if si.overall_before_local:
      # first do overall sharpening of the map to get it about right
      print(80*"*", file=out)
      print("\nSharpening map overall before carrying out local sharpening\n", file=out)
      overall_si=deepcopy(si)
      overall_si.local_sharpening=False  # don't local sharpen
      overall_si=auto_sharpen_map_or_map_coeffs(si=overall_si,
            auto_sharpen_methods=auto_sharpen_methods,
            map=map,
            half_map_data_list=half_map_data_list,
            pdb_inp=pdb_inp,
            out=out)
      sharpened_map=overall_si.map_data
      print("\nDone sharpening map overall before carrying out local sharpening\n", file=out)
      print(80*"*", file=out)
    else:
      sharpened_map=map
  
  
    # Accumulate sums
    starting_weight=0.01 # put starting map everywhere with low weight
    sum_weight_map=make_empty_map(template_map=map,value=starting_weight)
  
    # in case a pixel is not covered...
    sum_weight_value_map=starting_weight*sharpened_map.deep_copy()
  
    print("\nUsing overall map for any regions where "+\
       "no local information is present", file=out)
  
    id_list=[]
    b_iso_list=flex.double()
    starting_b_iso_list=flex.double()
  
    # use sharpened map here
    upper_bounds_list,lower_bounds_list,\
       centers_cart_ncs_list,centers_cart,all_cart=\
       get_target_boxes(si=si,map=sharpened_map,ncs_obj=ncs_obj,
         pdb_inp=pdb_inp,out=out)
  
    dist=mean_dist_to_nearest_neighbor(all_cart)
    if not dist:
      dist=10.
      if not si.smoothing_radius:
        print("No nearest neighbors...best to set smoothing radius", file=out)
    print("\nMean distance to nearest center is %7.2f A " %(
      dist), file=out)
    if not si.smoothing_radius:
      si.smoothing_radius=float("%.0f" %(dist*2/3)) # 10A from nearest neighbor
      print("Using %s A for smoothing radius" %(si.smoothing_radius), file=out)
  
    i=-1
    for ub,lb,centers_ncs_cart,center_cart in zip(
      upper_bounds_list,lower_bounds_list,centers_cart_ncs_list,centers_cart):
      i+=1
      if si.select_sharpened_map is not None and i != si.select_sharpened_map:
        continue
      map_file_name='sharpened_map_%s.ccp4' %(i)
      if 0 and si.read_sharpened_maps: # cannot do this as no bounds
        print("\nReading sharpened map directly from %s" %(map_file_name), file=out)
        result=get_map_object(file_name=map_file_name,
          out=out)
        local_map_data=result[0]
      else:
  
        local_si=deepcopy(si)
        local_si.local_sharpening=False  # don't do it again
        local_si.box_size=get_box_size(lower_bound=lb,upper_bound=ub)
        local_si.box_center=center_cart
        local_si.box_in_auto_sharpen=True
        local_si.density_select_in_auto_sharpen=False
        local_si.use_local_aniso=si.local_aniso_in_local_sharpening
        local_si.remove_aniso=si.local_aniso_in_local_sharpening
        local_si.max_box_fraction=999 # just bigger than 1
        local_si.density_select_max_box_fraction=999
        local_si.nproc=1
        print(80*"+", file=out)
        print("Getting local sharpening for box %s" %(i), file=out)
        print(80*"+", file=out)
        bsi=auto_sharpen_map_or_map_coeffs(si=local_si,
          auto_sharpen_methods=auto_sharpen_methods,
          map=sharpened_map,
          half_map_data_list=half_map_data_list,
          pdb_inp=pdb_inp,
          return_bsi=True, # just return the bsi of sharpened data
          out=out)
  
        if not bsi.map_data:
          print("\nNo result for local map %s ...skipping" %(i), file=out)
          continue
  
        # merge with background using bsi.smoothed_box_mask_data
        if bsi.smoothed_box_mask_data:
          print("Merging small map into overall map in soft-mask region", file=out)
          bsi.merge_into_overall_map(overall_map=map) # XXX overall_map not used
  
        # Now remove buffer region
        if bsi.n_buffer: # extract just the good part
           print("Removing buffer from small map", file=out)
           bsi.remove_buffer(out=out)
  
  
      weight_data=bsi.get_gaussian_weighting(out=out)
      weighted_data=bsi.map_data*weight_data
  
      sum_weight_value_map=sum_box_data(starting_map=sum_weight_value_map,
         box_map=weighted_data,
         lower_bounds=bsi.lower_bounds,
         upper_bounds=bsi.upper_bounds)
  
      sum_weight_map=sum_box_data(starting_map=sum_weight_map,
         box_map=weight_data,
         lower_bounds=bsi.lower_bounds,
         upper_bounds=bsi.upper_bounds)
  
      id_list.append(i)
      starting_b_iso_list.append(bsi.starting_b_iso)
      b_iso_list.append(bsi.b_iso)
  
      print(80*"+", file=out)
      print("End of getting local sharpening for small box %s" %(i), file=out)
      print(80*"+", file=out)
  
    print("\nOverall map created from total of %s local maps" %(i), file=out)
    if si.overall_before_local:
      print("Note: overall map already sharpened with global sharpening", file=out)
  
    if starting_b_iso_list.size()<1:
      print("No results for local sharpening...", file=out)
    else:
      print("Summary of b_iso values by local map:", file=out)
      print(" ID   Starting B-iso    Sharpened B-iso", file=out)
      for i,starting_b_iso,b_iso in zip(id_list,starting_b_iso_list,b_iso_list):
        print(" %4s    %7.2f        %7.2f" %(i,starting_b_iso,b_iso), file=out)
      print("\nMean    %7.2f        %7.2f" %(
       starting_b_iso_list.min_max_mean().mean,
       b_iso_list.min_max_mean().mean), file=out)
  
    si.map_data=sum_weight_value_map/sum_weight_map
  
    # Get overall b_iso...
    print("\nGetting overall b_iso of composite map...", file=out)
    map_coeffs_aa,map_coeffs,f_array,phases=effective_b_iso(
       map_data=si.map_data,
        resolution=si.resolution,
        d_min_ratio=si.d_min_ratio,
        scale_max=si.scale_max,
        crystal_symmetry=si.crystal_symmetry,
        out=out)
  
    print(80*"+", file=out)
    print("End of getting local sharpening ", file=out)
    print(80*"+", file=out)
  
    return si
  
  def auto_sharpen_map_or_map_coeffs(
          si=None,
          resolution=None,        # resolution is required
          crystal_symmetry=None,  # supply crystal_symmetry and map or
          map=None,               #  map and n_real
          half_map_data_list=None,     #  two half-maps matching map
          is_crystal=None,
          map_coeffs=None,
          pdb_inp=None,
          ncs_obj=None,
          seq_file=None,
          sequence=None,
          rmsd=None,
          rmsd_resolution_factor=None,
          k_sol=None,
          b_sol=None,
          fraction_complete=None,
          n_real=None,
          solvent_content=None,
          molecular_mass=None,
          region_weight=None,
          sa_percent=None,
          n_bins=None,
          eps=None,
          max_regions_to_test=None,
          regions_to_keep=None,
          fraction_occupied=None,
          input_weight_map_pickle_file=None,
          output_weight_map_pickle_file=None,
          read_sharpened_maps=None,
          write_sharpened_maps=None,
          select_sharpened_map=None,
          output_directory=None,
          smoothing_radius=None,
          local_sharpening=None,
          local_aniso_in_local_sharpening=None,
          overall_before_local=None,
          use_local_aniso=None,
          auto_sharpen=None,
          box_in_auto_sharpen=None, # n_residues, ncs_copies required if not False
          density_select_in_auto_sharpen=None,
          density_select_threshold_in_auto_sharpen=None,
          allow_box_if_b_iso_set=None,
          use_weak_density=None,
          discard_if_worse=None,
          n_residues=None,
          ncs_copies=None,
          box_center=None,
          remove_aniso=None,
          box_size=None,
          target_n_overlap=None,
          lower_bounds=None,
          upper_bounds=None,
          restrict_map_size=None,
          auto_sharpen_methods=None,
          residual_target=None,
          sharpening_target=None,
          d_min_ratio=None,
          scale_max=None,
          input_d_cut=None,
          b_blur_hires=None,
          max_box_fraction=None,
          cc_cut=None,
          max_cc_for_rescale=None,
          scale_using_last=None,
          density_select_max_box_fraction=None,
          mask_atoms=None,
          mask_atoms_atom_radius=None,
          value_outside_atoms=None,
          soft_mask=None,
          tol_r=None,
          abs_tol_t=None,
          rel_tol_t=None,
          require_helical_or_point_group_symmetry=None,
          max_helical_operators=None,
          k_sharpen=None,
          optimize_d_cut=None,
          optimize_b_blur_hires=None,
          iterate=None,
          search_b_min=None,
          search_b_max=None,
          search_b_n=None,
          adjust_region_weight=None,
          region_weight_method=None,
          region_weight_factor=None,
          region_weight_buffer=None,
          region_weight_default=None,
          target_b_iso_ratio=None,
          signal_min=None,
          buffer_radius=None,
          wang_radius=None,
          pseudo_likelihood=None,
          target_b_iso_model_scale=None,
          b_iso=None, # if set, use it
          b_sharpen=None, # if set, use it
          resolution_dependent_b=None, # if set, use it
          normalize_amplitudes_in_resdep=None, # if set, use it
          return_bsi=False,
          verbose=None,
          resolve_size=None,
          nproc=None,
          multiprocessing=None,
          queue_run_command=None,
          out=sys.stdout):
      if si:  #
        resolution=si.resolution
        crystal_symmetry=si.crystal_symmetry
        if not auto_sharpen:
          auto_sharpen=si.auto_sharpen
        if verbose is None:
          verbose=si.verbose
        if resolve_size is None:
          resolve_size=si.resolve_size
  
      if auto_sharpen is None:
        auto_sharpen=True
  
      if map_coeffs and not resolution:
         resolution=map_coeffs.d_min()
      if map_coeffs and not crystal_symmetry:
         crystal_symmetry=map_coeffs.crystal_symmetry()
  
      assert resolution is not None
  
      if map:
        return_as_map=True
      else:  # convert from structure factors to create map if necessary
        map=get_fft_map(n_real=n_real, map_coeffs=map_coeffs).real_map_unpadded()
        return_as_map=False
  
      # Set ncs_copies if possible
      if ncs_copies is None and ncs_obj and ncs_obj.max_operators():
        ncs_copies=ncs_obj.max_operators()
        print("Set ncs copies based on ncs_obj to %s" %(ncs_copies), file=out)
  
      # Determine if we are running model_sharpening
      if half_map_data_list and len(half_map_data_list)==2:
        auto_sharpen_methods=['half_map_sharpening']
      elif pdb_inp:
        auto_sharpen_methods=['model_sharpening']
      if not si:
        # Copy parameters to si (sharpening_info_object)
        si=set_up_si(var_dict=locals(),
          crystal_symmetry=crystal_symmetry,
          is_crystal=is_crystal,
          solvent_fraction=solvent_content,
          molecular_mass=molecular_mass,
          auto_sharpen=auto_sharpen,
          map=map,
          verbose=verbose,
          half_map_data_list=half_map_data_list,
          pdb_inp=pdb_inp,
          ncs_copies=ncs_copies,
          n_residues=n_residues,out=out)
      # Figure out solvent fraction
      if si.solvent_fraction is None:
        si.solvent_fraction=get_iterated_solvent_fraction(
          crystal_symmetry=crystal_symmetry,
          verbose=si.verbose,
          resolve_size=si.resolve_size,
          fraction_of_max_mask_threshold=si.fraction_of_max_mask_threshold,
          mask_resolution=si.resolution,
          map=map,
          out=out)
      if si.solvent_fraction:
        print("Estimated solvent content: %.2f" %(si.solvent_fraction), file=out)
      else:
        raise Sorry("Unable to estimate solvent content...please supply "+
          "solvent_content \nor molecular_mass")
      # Determine if we are running half-map or model_sharpening
      if half_map_data_list and len(half_map_data_list)==2:
        first_half_map_data=half_map_data_list[0]
        second_half_map_data=half_map_data_list[1]
      else:
        first_half_map_data=None
        second_half_map_data=None
  
      # Decide if we are running local sharpening (overlapping set of sharpening
      #   runs at various locations)
      libtbx.call_back(message='sharpen',data=None)
      if si.local_sharpening:
        return run_local_sharpening(si=si,
           auto_sharpen_methods=auto_sharpen_methods,
           map=map,
           ncs_obj=ncs_obj,
           half_map_data_list=half_map_data_list,
           pdb_inp=pdb_inp,
           out=out)
  
      # Get preliminary values of sharpening
      working_map=map  # use another name for map XXX
      if si.iterate and not si.preliminary_sharpening_done:
        si.preliminary_sharpening_done=True
        si.iterate=False
        # first do overall sharpening of the map to get it about right
        print(80*"*", file=out)
        print("\nSharpening map overall before carrying out final sharpening\n", file=out)
        overall_si=deepcopy(si)
        overall_si.local_sharpening=False  # don't local sharpen
        overall_si=auto_sharpen_map_or_map_coeffs(si=overall_si,
              auto_sharpen_methods=auto_sharpen_methods,
              map=map,
              half_map_data_list=half_map_data_list,
              pdb_inp=pdb_inp,
              out=out)
        working_map=overall_si.map_data
        # Get solvent content again
        overall_si.solvent_content=None
        overall_si.solvent_fraction=get_iterated_solvent_fraction(
          crystal_symmetry=crystal_symmetry,
          verbose=overall_si.verbose,
          resolve_size=overall_si.resolve_size,
          fraction_of_max_mask_threshold=si.fraction_of_max_mask_threshold,
          mask_resolution=si.resolution,
          map=working_map,
          out=out)
        print("Resetting solvent fraction to %.2f " %(
           overall_si.solvent_fraction), file=out)
        si.solvent_fraction=overall_si.solvent_fraction
  
        print("\nDone sharpening map overall before carrying out final sharpening\n", file=out)
        print(80*"*", file=out)
        si.b_blur_hires=0.  # from now on, don't apply extra blurring
      else:
        working_map=map
  
      # Now identify optimal sharpening params
      print(80*"=", file=out)
      print("\nRunning auto_sharpen to get sharpening parameters\n", file=out)
      print(80*"=", file=out)
      result=run_auto_sharpen( # get sharpening parameters standard run
        si=si,
        map_data=working_map,
        first_half_map_data=first_half_map_data,
        second_half_map_data=second_half_map_data,
        pdb_inp=pdb_inp,
        auto_sharpen_methods=auto_sharpen_methods,
        print_result=False,
        return_bsi=return_bsi,
        out=out)
      if return_bsi:
        return result  # it is a box_sharpening_info object
      else:
        si=result
      print(80*"=", file=out)
      print("\nDone running auto_sharpen to get sharpening parameters\n", file=out)
      print(80*"=", file=out)
  
      # Apply the optimal sharpening values and save map in si.map_data
      # First test without sharpening if sharpening_method is b_iso,b and
      # b_iso is not set
      if si.sharpening_method in [
         'b_iso','b_iso_to_d_cut','resolution_dependent'] and b_iso is None:
        local_si=deepcopy(si)
        local_si.sharpening_method='no_sharpening'
        local_si.sharpen_and_score_map(map_data=working_map,out=null_out())
        print("\nScore for no sharpening: %7.2f " %(local_si.score), file=out)
      else:
        local_si=None
  
      print(80*"=", file=out)
      print("\nApplying final sharpening to entire map", file=out)
      print(80*"=", file=out)
      si.sharpen_and_score_map(map_data=working_map,set_b_iso=True,out=out)
      if si.discard_if_worse and local_si and local_si.score > si.score:
         print("Sharpening did not improve map "+\
          "(%7.2f sharpened, %7.2f unsharpened). Discarding sharpened map" %(
          si.score,local_si.score), file=out)
         print("\nUse discard_if_worse=False to keep the sharpening", file=out)
         local_si.sharpen_and_score_map(map_data=working_map,out=out)
         si=local_si
      if not si.is_model_sharpening() and not si.is_half_map_sharpening():
        si.show_score(out=out)
        si.show_summary(out=out)
  
      return si  # si.map_data is the sharpened map
  
  def estimate_signal_to_noise(value_list=None,minimum_value_to_include=0):
    # get "noise" from rms value of value_list(n) compared with average of n-2,n-1,n+1,n+2
    # assumes middle is the high part of the very smooth curve.
    # Don't include values < minimum_value_to_include
    mean_square_diff=0.
    mean_square_diff_n=0.
    for b2,b1,value,p1,p2 in zip(value_list,
      value_list[1:],
      value_list[2:],
      value_list[3:],
      value_list[4:]):
      too_low=False
      for xx in [b2,b1,value,p1,p2]:
        if xx <minimum_value_to_include:
          too_low=True
      if not too_low:
        mean_square_diff_n+=1
        mean_square_diff+=( (b2+b1+p1+p2)*0.25 - value)**2
    rmsd=(mean_square_diff/max(1,mean_square_diff_n))**0.5
    if value_list.size()>0:
      min_adj_sa=max(value_list[0],value_list[-1])
      max_adj_sa=value_list.min_max_mean().max
      signal_to_noise=(max_adj_sa-min_adj_sa)/max(1.e-10,rmsd)
    else:
      signal_to_noise=0.
    return signal_to_noise
  
  def optimize_b_blur_or_d_cut_or_b_iso(
             optimization_target='b_blur_hires',
             local_best_si=None,
             local_best_map_and_b=None,
             si_id_list=None,
             si_score_list=None,
             delta_b=None,
             original_b_iso=None,
             f_array=None,
             phases=None,
             delta_b_blur_hires=100,
             delta_d_cut=0.25,
             n_cycle_optimize=5,
             min_cycles=2,
             n_range=5,
             out=sys.stdout):
  
    assert optimization_target in ['b_blur_hires','d_cut','b_iso']
    if optimization_target=='b_blur_hires':
      print("\nOptimizing b_blur_hires. ", file=out)
    elif optimization_target=='d_cut':
      print("\nOptimizing d_cut. ", file=out)
      local_best_si.input_d_cut=local_best_si.get_d_cut()
    elif optimization_target=='b_iso':
      print("\nOptimizing b_iso. ", file=out)
  
    local_best_si.show_summary(out=out)
  
    print("Current best score=%7.3f b_iso=%5.1f  b_blur_hires=%5.1f d_cut=%5.1f" %(
         local_best_si.score,local_best_si.b_iso,
         local_best_si.b_blur_hires,
         local_best_si.get_d_cut()), file=out)
  
  
    # existing values:
    value_dict={}
    for id,score in zip(si_id_list,si_score_list):
      value_dict[id]=score
    best_score=local_best_si.score
    delta_b_iso=delta_b
  
    local_best_score=best_score
    improving=True
    working_best_si=deepcopy(local_best_si)
    for cycle in range(n_cycle_optimize):
      if not improving: break
      print("Optimization cycle %s" %(cycle), file=out)
      print("Current best score=%7.3f b_iso=%5.1f  b_blur_hires=%5.1f d_cut=%5.1f" %(
       working_best_si.score,working_best_si.b_iso,
          working_best_si.b_blur_hires,
         working_best_si.get_d_cut()), file=out)
      if working_best_si.verbose:
        print(" B-sharpen B-iso B-blur   Adj-SA    "+\
             "Kurtosis  SA-ratio   Regions   d_cut   b_blur_hires", file=out)
      local_best_working_si=deepcopy(working_best_si)
      improving=False
      for jj in range(-n_range,n_range+1):
          if optimization_target=='b_blur_hires': # ZZ try optimizing b_blur_hires
            test_b_blur_hires=max(0.,working_best_si.b_blur_hires+jj*delta_b_blur_hires)
            test_d_cut=working_best_si.get_d_cut()
            test_b_iso=working_best_si.b_iso
          elif optimization_target=='d_cut':
            test_b_blur_hires=working_best_si.b_blur_hires
            test_d_cut=working_best_si.get_d_cut()+jj*delta_d_cut
            test_b_iso=working_best_si.b_iso
          elif optimization_target=='b_iso':
            test_b_blur_hires=working_best_si.b_blur_hires
            test_d_cut=working_best_si.get_d_cut()
            test_b_iso=working_best_si.b_iso+jj*delta_b_iso
  
          id="%.3f_%.3f_%.3f" %(test_b_iso,test_b_blur_hires,test_d_cut)
          if id in value_dict:
            score=value_dict[id]
          else:
            local_si=deepcopy(local_best_si)
            local_f_array=f_array
            local_phases=phases
            local_si.b_blur_hires=test_b_blur_hires
            local_si.input_d_cut=test_d_cut
            local_si.b_iso=test_b_iso
            local_si.b_sharpen=original_b_iso-local_si.b_iso
  
            local_map_and_b=apply_sharpening(
              f_array=local_f_array,phases=local_phases,
              sharpening_info_obj=local_si,
              crystal_symmetry=local_si.crystal_symmetry,
              out=null_out())
            local_si=score_map(
               map_data=local_map_and_b.map_data,sharpening_info_obj=local_si,
              out=null_out())
            value_dict[id]=local_si.score
            if local_si.verbose:
              print(" %6.1f     %6.1f  %5s   %7.3f  %7.3f" %(
                local_si.b_sharpen,local_si.b_iso,
                 local_si.b_blur_hires,
                 local_si.adjusted_sa,local_si.kurtosis) + \
                "  %7.3f         %7.3f   %7.3f %7.3f " %(
                 local_si.sa_ratio,local_si.normalized_regions,
                 test_d_cut,
                 test_b_blur_hires
                  ), file=out)
  
            if local_si.score > local_best_score:
              local_best_score=local_si.score
              local_best_working_si=deepcopy(local_si)
      if local_best_score > best_score:
        best_score=local_best_score
        working_best_si=deepcopy(local_best_working_si)
        delta_b_iso=delta_b_iso/2
        delta_b_blur_hires=delta_b_blur_hires/2
        delta_d_cut=delta_d_cut/2
        print("Current working best "+\
            "score=%7.3f b_iso=%5.1f  b_blur_hires=%5.1f d_cut=%5.1f" %(
              working_best_si.score,working_best_si.b_iso,
          working_best_si.b_blur_hires,
          working_best_si.get_d_cut()), file=out)
        improving=True
  
    if working_best_si and working_best_si.score > local_best_si.score:
      print("Using new values of b_iso and b_blur_hires and d_cut", file=out)
      local_best_si=working_best_si
  
    local_best_si.show_summary(out=out)
  
    return local_best_si,local_best_map_and_b
  
  def set_mean_sd_of_map(map_data=None,target_mean=None,target_sd=None):
      if not map_data: return None
      new_mean=map_data.as_1d().min_max_mean().mean
      new_sd=max(1.e-10,map_data.sample_standard_deviation())
      map_data=(map_data-new_mean)/new_sd # normalized
      return map_data*target_sd + target_mean # restore original
  
  def run_auto_sharpen(
        si=None,
        map_data=None,
        first_half_map_data=None,
        second_half_map_data=None,
        pdb_inp=None,
        auto_sharpen_methods=None,
        print_result=True,
        return_bsi=False,
        out=sys.stdout):
  
    if si.verbose:
       local_out=out
    else:
       local_out=null_out()
    #  Identifies parameters for optimal map sharpening using analysis of density,
    #    model-correlation, or half-map correlation (first_half_map_data vs
    #     vs second_half_map_data).
  
    #  NOTE: We can apply this to any map_data (a part or whole of the map)
    #  BUT: need to update n_real if we change the part of the map!
    #  change with map data: crystal_symmetry, solvent_fraction, n_real, wrapping,
  
    smoothed_box_mask_data=None
    original_box_map_data=None
    if si.auto_sharpen and (
      si.box_in_auto_sharpen or si.density_select_in_auto_sharpen or pdb_inp):
  
      original_box_sharpening_info_obj=deepcopy(si)  # should really not be box
      box_pdb_inp,box_map_data,box_first_half_map_data,\
           box_second_half_map_data,\
           box_crystal_symmetry,box_sharpening_info_obj,\
           smoothed_box_mask_data,original_box_map_data,n_buffer=\
         select_box_map_data(si=si,
             map_data=map_data,
             first_half_map_data=first_half_map_data,
             second_half_map_data=second_half_map_data,
             pdb_inp=pdb_inp,
             restrict_map_size=si.restrict_map_size,
             out=out,local_out=local_out)
  
      if box_sharpening_info_obj is None: # did not do it
        print("Box map is similar in size to entire map..."+\
           "skipping representative box of density", file=out)
        original_box_sharpening_info_obj=None
        crystal_symmetry=si.crystal_symmetry
      else:
        print("Using small map to identify optimal sharpening", file=out)
        print("Box map grid: %d  %d  %d" %(
           box_map_data.all()), file=out)
        print("Box map cell: %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f  "%(
          box_crystal_symmetry.unit_cell().parameters()), file=out)
        original_map_data=map_data
        original_crystal_symmetry=si.crystal_symmetry
  
        map_data=box_map_data
        pdb_inp=box_pdb_inp
        if si.density_select_in_auto_sharpen and ( # catch empty pdb_inp
           not pdb_inp or not \
             pdb_inp.construct_hierarchy().overall_counts().n_residues):
          pdb_inp=None
  
        crystal_symmetry=box_crystal_symmetry
        if box_first_half_map_data:
          first_half_map_data=box_first_half_map_data
        if box_second_half_map_data:
          second_half_map_data=box_second_half_map_data
        # SET si for box now...
        si=deepcopy(si).update_with_box_sharpening_info(
          box_sharpening_info_obj=box_sharpening_info_obj)
    else:
      original_box_sharpening_info_obj=None
      box_sharpening_info_obj=None
      crystal_symmetry=si.crystal_symmetry
  
    starting_mean=map_data.as_1d().min_max_mean().mean
    starting_sd=map_data.sample_standard_deviation()
  
    print("\nGetting original b_iso...", file=out)
    map_coeffs_aa,map_coeffs,f_array,phases=effective_b_iso(
       map_data=map_data,
        resolution=si.resolution,
        d_min_ratio=si.d_min_ratio,
        scale_max=si.scale_max,
        remove_aniso=si.remove_aniso,
        crystal_symmetry=si.crystal_symmetry,
        out=out)
    original_b_iso=map_coeffs_aa.b_iso
    if original_b_iso is None:
      print("Could not determine original b_iso...setting to 200", file=out)
      original_b_iso=200.
  
    si.original_aniso_obj=map_coeffs_aa # set it so we can apply it later if desired
  
    if first_half_map_data:
      first_half_map_coeffs,dummy=get_f_phases_from_map(
            map_data=first_half_map_data,
         crystal_symmetry=si.crystal_symmetry,
         d_min=si.resolution,
         d_min_ratio=si.d_min_ratio,
         remove_aniso=si.remove_aniso,
         scale_max=si.scale_max,
         return_as_map_coeffs=True,
         out=local_out)
    else:
      first_half_map_coeffs=None
  
    if second_half_map_data:
      second_half_map_coeffs,dummy=get_f_phases_from_map(
        map_data=second_half_map_data,
         crystal_symmetry=si.crystal_symmetry,
         d_min=si.resolution,
         d_min_ratio=si.d_min_ratio,
         scale_max=si.scale_max,
         remove_aniso=si.remove_aniso,
         return_as_map_coeffs=True,
         out=local_out)
    else:
      second_half_map_coeffs=None
  
    if pdb_inp:
      # Getting model information if pdb_inp present ---------------------------
      from cctbx.maptbx.refine_sharpening import get_model_map_coeffs_normalized
      model_map_coeffs=get_model_map_coeffs_normalized(pdb_inp=pdb_inp,
         si=si,
         f_array=f_array,
         resolution=si.resolution,
         out=out)
    else:
      model_map_coeffs=None
  
    # Try various methods for sharpening. # XXX fix this up
  
    local_si=deepcopy(si).update_with_box_sharpening_info(
        box_sharpening_info_obj=box_sharpening_info_obj)
    if si.adjust_region_weight and \
        (not si.sharpening_is_defined()) and (not si.is_model_sharpening()) \
       and (not si.is_half_map_sharpening()) and (
        not si.is_target_b_iso_to_d_cut()) and (
        si.sharpening_target=='adjusted_sa'):
     for iii in range(1):  # just so we can break
  
      local_si=deepcopy(si).update_with_box_sharpening_info(
        box_sharpening_info_obj=box_sharpening_info_obj)
      local_si.sharpening_target='adjusted_sa'
      local_si.sharpening_method='b_iso_to_d_cut'
  
  
      sa_ratio_list=[]
      normalized_regions_list=[]
  
      if 0: #si.resolution:
        # 2017-07-26 reset b_low,b_mid,b_high, using 5.9*resolution**2 for b_mid
        delta_search=si.search_b_max-si.search_b_min
        b_mid=si.get_target_b_iso()
        b_low=b_mid-150*delta_search/400
        b_high=b_mid+250*delta_search/400
        print("Centering search on b_iso=%7.2f" %(b_mid), file=out)
      else:
        b_low=min(original_b_iso,si.search_b_min)
        b_high=max(original_b_iso,si.search_b_max)
        b_mid=b_low+0.375*(b_high-b_low)
  
      ok_region_weight=True
      results_list=[]
      kw_list=[]
      first=True
  
      id=0
      for b_iso in [b_low,b_high,b_mid]:
        id+=1
  
        if first and local_si.multiprocessing=='multiprocessing' or \
            local_si.nproc==1:  # can do anything
          local_log=out
        else:  # skip log entirely
          local_log=None  # will set this later and return as r.log_as_text
        first=False
        lsi=deepcopy(local_si)
  
        lsi.b_sharpen=original_b_iso-b_iso
        lsi.b_iso=b_iso
  
        # ------ SET UP RUN HERE ----------
        kw_list.append(
        {
        'f_array':f_array,
        'phases':phases,
          'crystal_symmetry':lsi.crystal_symmetry,
          'local_si':lsi,
          'id':id,
          'out':local_log,
         })
        # We are going to call autosharpening with this
        # ------ END OF SET UP FOR RUN ----------
  
        """
  
        local_map_and_b=apply_sharpening(
              f_array=f_array,phases=phases,
              sharpening_info_obj=lsi,
              crystal_symmetry=lsi.crystal_symmetry,
              out=null_out())
        local_si=score_map(map_data=local_map_and_b.map_data,
            sharpening_info_obj=local_si,
            out=null_out())
        """
        # This is the actual run here =============
  
      from libtbx.easy_mp import run_parallel
      results_list=run_parallel(
         method=si.multiprocessing,
         qsub_command=si.queue_run_command,
         nproc=si.nproc,
         target_function=run_sharpen_and_score,kw_list=kw_list)
      # results looks like: [result,result2]
  
      sort_list=[]
      for result in results_list:
        sort_list.append([result.id,result])
      sort_list.sort()
      for id,result in sort_list:
        local_si=result.local_si
  
        if local_si.sa_ratio is None or local_si.normalized_regions is None:
          ok_region_weight=False
        sa_ratio_list.append(local_si.sa_ratio)
        normalized_regions_list.append(local_si.normalized_regions)
      if not ok_region_weight:
        break # skip it
  
      # Set region weight so that either:
      #  (1) delta_sa_ratio==region_weight*delta_normalized_regions
      #  (2) sa_ratio=region_weight*normalized_regions (at low B)
  
      # region weight from change over entire region
      d_sa_ratio=sa_ratio_list[0]-sa_ratio_list[1]
      d_normalized_regions=normalized_regions_list[0]-normalized_regions_list[1]
      delta_region_weight=si.region_weight_factor*d_sa_ratio/max(
            1.e-10,d_normalized_regions)
      if d_sa_ratio < 0 or d_normalized_regions < 0:
        print("Not using delta_region_weight with unusable values", file=out)
        ok_region_weight=False
  
  
      # region weight from initial values
      init_region_weight=si.region_weight_factor* \
            sa_ratio_list[0]/max(1.e-10,normalized_regions_list[0])
  
      # Ensure that adjusted_sa at b_mid is > than either end
      #  adjusted_sa=sa_ratio - region_weight*normalized_regions
  
      # sa[2]=sa_ratio_list[2]-region_weight*normalized_regions[2]
      # sa[1]=sa_ratio_list[1]-region_weight*normalized_regions[1]
      # sa[0]=sa_ratio_list[0]-region_weight*normalized_regions[0]
      #  sa[2] >= sa[1] and sa[2] >= sa[0]
      # sa_ratio_list[2]-region_weight*normalized_regions[2] >=
      #    sa_ratio_list[1]-region_weight*normalized_regions[1]
      # NOTE: sa_ratio_list and normalized_regions both  decrease in order:
      #     low med high or [0] [2] [1]
      max_region_weight = (sa_ratio_list[2]- sa_ratio_list[1])/max(0.001,
           normalized_regions_list[2]-normalized_regions_list[1])
      min_region_weight = (sa_ratio_list[0]- sa_ratio_list[2])/max(0.001,
         normalized_regions_list[0]-normalized_regions_list[2])
  
      min_region_weight=max(1.e-10,min_region_weight) # positive
      max_region_weight=max(1.e-10,max_region_weight) # positive
  
      delta_weight=max(0.,max_region_weight-min_region_weight)
      min_buffer=delta_weight*si.region_weight_buffer
      min_region_weight+=min_buffer
      max_region_weight-=min_buffer
      min_max_region_weight=True
      if min_region_weight >= max_region_weight:
        print("Warning: min_region_weight >= max_region_weight...", file=out)
        min_max_region_weight=False
        #ok_region_weight=False
  
  
      print("Region weight bounds: Min: %7.1f  Max: %7.1f " %(
        min_region_weight,max_region_weight), file=out)
      print("Region weight estimates:", file=out)
      print("From ratio of low-B surface area to regions: %7.1f" %(
       init_region_weight), file=out)
      print("Ratio of change in surface area to change in regions: %7.1f" %(
       delta_region_weight), file=out)
  
      # put them in bounds but note if we did it
  
      out_of_range=False
      if ok_region_weight and si.region_weight_method=='initial_ratio':
        if min_max_region_weight and (
            init_region_weight > max_region_weight or \
            init_region_weight<min_region_weight):
          init_region_weight=max(
            min_region_weight,min(max_region_weight,init_region_weight))
          out_of_range=True
        print("\nRegion weight adjusted to %7.1f using initial ratio" %(
            init_region_weight), file=out)
        si.region_weight=init_region_weight
  
      elif ok_region_weight and si.region_weight_method=='delta_ratio':
        if min_max_region_weight and (
            delta_region_weight > max_region_weight or \
            delta_region_weight<min_region_weight):
          delta_region_weight=max(
            min_region_weight,min(max_region_weight,delta_region_weight))
          out_of_range=True
        si.region_weight=delta_region_weight
        print("\nRegion weight set to %7.1f using overall ratio and " %(
            si.region_weight) +\
            "\nfactor of %5.1f" %(si.region_weight_factor), file=out)
  
      else: # just use default target for b_iso
        si.region_weight=si.region_weight_default
  
        print("Skipping region_weight analysis as signal-to-noise is zero ("+\
             "adjusted sa\nvs b_iso does not have low values at extremes and "+\
             "clear maximum in the middle.)", file=out)
  
        print("\nUnable to set region_weight ... using value of %7.2f" % (
            si.region_weight), file=out)
        if si.discard_if_worse:
          print("Setting discard_if_worse=False as region_weight failed ", file=out)
          si.discard_if_worse=False
  
      if out_of_range and 'resolution_dependent' in auto_sharpen_methods:
        new_list=[]
        have_something_left=False
        for x in auto_sharpen_methods:
          if x != 'resolution_dependent':
            if str(x) != 'None':
              have_something_left=True
            new_list.append(x)
        if have_something_left:
          auto_sharpen_methods=new_list
          print("Removed resolution_dependent sharpening ( "+\
            "weights were out of range)", file=out)
  
      if box_sharpening_info_obj:
        si.local_solvent_fraction=box_sharpening_info_obj.solvent_fraction
      else:
        si.local_solvent_fraction=si.solvent_fraction
  
  
    null_si=None
    best_si=deepcopy(si).update_with_box_sharpening_info(
        box_sharpening_info_obj=box_sharpening_info_obj)
    best_map_and_b=map_and_b_object()
  
    if si.sharpening_is_defined():  # Use this if come in with method
      print("\nUsing specified sharpening", file=out)
      best_si=set_up_sharpening(si=si,map_data=map_data,out=out)
      best_si.sharpen_and_score_map(map_data=map_data,
            out=out).show_score(out=out)
      best_si.show_summary(out=out)
  
    else:
      if best_si.is_model_sharpening():
        print("\nSetting up model sharpening", file=out)
      elif best_si.is_half_map_sharpening():
        print("\nSetting up half-map sharpening", file=out)
      else:
        print("\nTesting sharpening methods with target of %s" %(
          best_si.sharpening_target), file=out)
      if not auto_sharpen_methods or auto_sharpen_methods==['None']:
        auto_sharpen_methods=['no_sharpening']
      for m in auto_sharpen_methods:
        # ------------------------
        if m in ['no_sharpening','resolution_dependent','model_sharpening',
            'half_map_sharpening','target_b_iso_to_d_cut']:
          if m=='target_b_iso_to_d_cut':
            b_min=si.get_target_b_iso()
            b_max=si.get_target_b_iso()
          else:
            b_min=original_b_iso
            b_max=original_b_iso
          b_n=1
          k_sharpen=0.
          delta_b=0
          if m in ['resolution_dependent','model_sharpening',
             'half_map_sharpening']:
            pass # print out later
          else:
            print("\nB-sharpen   B-iso   k_sharpen   SA   "+\
               "Kurtosis  sa_ratio  Normalized regions", file=out)
        # ------------------------
        # ------------------------
        else:  #  ['b_iso','b_iso_to_d_cut']:
          if si.search_b_n>1:
            b_min=min(original_b_iso,si.search_b_min)
            b_max=max(original_b_iso,si.search_b_max)
          else: # for just one, take it
            b_min=si.search_b_min
            b_max=si.search_b_max
          b_n=si.search_b_n
          delta_b=(b_max-b_min)/max(1,b_n-1)
          print("\nTesting %s with b_iso from %7.1f to %7.1f in %d steps of %7.1f" %(
            m,b_min,b_max,b_n,delta_b), file=out)
          print("(b_sharpen from %7.1f to %7.1f ) " %(
             original_b_iso-b_min,original_b_iso-b_max), file=out)
          if m=='b_iso':
            k_sharpen=0.
          else:
            k_sharpen=si.k_sharpen
  
          print("\nB-sharpen   B-iso   k_sharpen   SA   "+\
               "Kurtosis  sa_ratio  Normalized regions", file=out)
        # ------------------------
        local_best_map_and_b=map_and_b_object()
        local_best_si=deepcopy(si).update_with_box_sharpening_info(
          box_sharpening_info_obj=box_sharpening_info_obj)
  
        si_b_iso_list=flex.double()
        si_score_list=flex.double()
        si_id_list=[]
  
        kw_list=[]
        first=True
        if return_bsi: assert local_si.nproc==1
  
        results_list=[]
        for i in range(b_n):
          # ============================================
          local_si=deepcopy(si).update_with_box_sharpening_info(
            box_sharpening_info_obj=box_sharpening_info_obj)
          local_si.sharpening_method=m
          local_si.n_real=map_data.all()
          local_si.k_sharpen=k_sharpen
  
          if first and local_si.multiprocessing=='multiprocessing' or \
              local_si.nproc==1:  # can do anything
            local_log=out
          else:  # skip log entirely
            local_log=None  # will set this later and return as r.log_as_text
          first=False
  
  
          if m=='resolution_dependent':
            print("\nRefining resolution-dependent sharpening based on %s" %(
              local_si.residual_target), file=out)
            local_si.b_sharpen=0
            local_si.b_iso=original_b_iso
            from cctbx.maptbx.refine_sharpening import run as refine_sharpening
            local_f_array,local_phases=refine_sharpening(
               map_coeffs=map_coeffs,
               sharpening_info_obj=local_si,
               out=out)
          elif m=='model_sharpening':
            print("\nUsing model-based sharpening", file=out)
            local_si.b_sharpen=0
            local_si.b_iso=original_b_iso
            from cctbx.maptbx.refine_sharpening import scale_amplitudes
            scale_amplitudes(
              model_map_coeffs=model_map_coeffs,map_coeffs=map_coeffs,
              si=local_si,out=out)
  
            # local_si contains target_scale_factors now
            local_f_array=f_array
            local_phases=phases
          elif m=='half_map_sharpening':
            print("\nUsing half-map-based sharpening", file=out)
            local_si.b_sharpen=0
            local_si.b_iso=original_b_iso
            from cctbx.maptbx.refine_sharpening import scale_amplitudes
            scale_amplitudes(
              model_map_coeffs=model_map_coeffs,
              map_coeffs=map_coeffs,
              first_half_map_coeffs=first_half_map_coeffs,
              second_half_map_coeffs=second_half_map_coeffs,
              si=local_si,out=out)
            # local_si contains target_scale_factors now
            local_f_array=f_array
            local_phases=phases
  
          else:
            local_f_array=f_array
            local_phases=phases
            b_iso=b_min+i*delta_b
            local_si.b_sharpen=original_b_iso-b_iso
            local_si.b_iso=b_iso
  
          # ------ SET UP RUN HERE ----------
          kw_list.append(
          {
          'f_array':local_f_array,
          'phases':local_phases,
            'crystal_symmetry':local_si.crystal_symmetry,
            'original_b_iso':original_b_iso,
            'local_si':local_si,
            'm':m,
            'return_bsi':return_bsi,
            'out':local_log,
            'id':i+1,
           })
          # We are going to call autosharpening with this
          # ------ END OF SET UP FOR RUN ----------
  
  
          # This is the actual run here =============
  
        from libtbx.easy_mp import run_parallel
        results_list=run_parallel(
           method=si.multiprocessing,
           qsub_command=si.queue_run_command,
           nproc=si.nproc,
           target_function=run_sharpen_and_score,kw_list=kw_list)
        # results looks like: [result,result2]
  
        sort_list=[]
        for result in results_list:
          sort_list.append([result.id,result])
        sort_list.sort()
        for id,result in sort_list:
          local_si=result.local_si
          local_map_and_b=result.local_map_and_b
          if result.text:
            print(result.text)
          # Run through all result to get these
          if local_si.b_sharpen is not None and local_si.b_iso is not None and\
             local_si.k_sharpen is not None and local_si.kurtosis is not None \
             and local_si.adjusted_sa is not None and local_si.score is not None:
              si_b_iso_list.append(local_si.b_iso)
              si_score_list.append(local_si.score)
              if local_si.k_sharpen is not None:
               si_id_list.append("%.3f_%.3f_%.3f" %(
                 local_si.b_iso,local_si.k_sharpen,
                 local_si.get_d_cut()))
  
          if m=='no_sharpening':
            null_si=local_si
          if local_best_si.score is None or local_si.score>local_best_si.score:
            local_best_si=local_si
            local_best_map_and_b=local_map_and_b
  
          # ============================================
          # DONE WITH ALL RUNS
  
        if not local_best_si.is_model_sharpening() and \
            not local_best_si.is_half_map_sharpening():
          if local_best_si.sharpening_method=='resolution_dependent':
            print("\nBest scores for sharpening with "+\
              "b[0]=%6.2f b[1]=%6.2f b[2]=%6.2f: " %(
              local_best_si.resolution_dependent_b[0],
              local_best_si.resolution_dependent_b[1],
              local_best_si.resolution_dependent_b[2]), file=out)
          else:
            print("\nBest scores for sharpening with "+\
              "b_iso=%6.1f b_sharpen=%6.1f k_sharpen=%s: " %(
              local_best_si.b_iso,local_best_si.b_sharpen,
               local_best_si.k_sharpen), file=out)
          if local_best_si.score is not None:
            local_best_si.show_summary(out=out)
            print("Adjusted surface area: %7.3f  Kurtosis: %7.3f  Score: %7.3f\n" %(
             local_best_si.adjusted_sa,local_best_si.kurtosis,local_best_si.score), file=out)
  
        if  si_score_list.size()>1: # test for signal
          signal_to_noise=estimate_signal_to_noise(value_list=si_score_list)
          print("Estimated signal-to-noise in ID of optimal sharpening: %5.1f" %(
             signal_to_noise), file=out)
          if signal_to_noise<local_best_si.signal_min and \
              'target_b_iso_to_d_cut' in auto_sharpen_methods:
            print("Skipping this analysis as signal-to-noise is less than %5.1f " %(
             local_best_si.signal_min), file=out)
            local_best_si.score=None
  
        optimize_b_blur_hires=False
        optimize_d_cut=False
        n_cycles=0
        if local_best_si.score is not None and local_best_si.optimize_d_cut and \
          local_best_si.sharpening_method in ['b_iso_to_d_cut','b_iso']:
          optimize_d_cut=True
          n_cycles+=1
        if local_best_si.score is not None and \
          local_best_si.optimize_b_blur_hires and \
          local_best_si.k_sharpen is not None and \
          local_best_si.sharpening_method in ['b_iso_to_d_cut','b_iso']:
          optimize_b_blur_hires=True
          n_cycles+=1
  
        ##########################################
        optimize_b_iso=True
        for cycle in range(n_cycles):
          if optimize_b_blur_hires:
            local_best_si,local_best_map_and_b=optimize_b_blur_or_d_cut_or_b_iso(
             #optimization_target='k_sharpen',
             optimization_target='b_blur_hires',
             local_best_si=local_best_si,
             local_best_map_and_b=local_best_map_and_b,
             si_id_list=si_id_list,
             si_score_list=si_score_list,
             delta_b=delta_b,
             original_b_iso=original_b_iso,
             f_array=f_array,
             phases=phases,
             out=out)
  
          if optimize_d_cut:
            local_best_si,local_best_map_and_b=optimize_b_blur_or_d_cut_or_b_iso(
             optimization_target='d_cut',
             local_best_si=local_best_si,
             local_best_map_and_b=local_best_map_and_b,
             si_id_list=si_id_list,
             si_score_list=si_score_list,
             delta_b=delta_b,
             original_b_iso=original_b_iso,
             f_array=f_array,
             phases=phases,
             out=out)
  
          if optimize_b_iso:
            local_best_si,local_best_map_and_b=optimize_b_blur_or_d_cut_or_b_iso(
             optimization_target='b_iso',
             local_best_si=local_best_si,
             local_best_map_and_b=local_best_map_and_b,
             si_id_list=si_id_list,
             si_score_list=si_score_list,
             delta_b=delta_b,
             original_b_iso=original_b_iso,
             f_array=f_array,
             phases=phases,
             out=out)
  
        ##########################################
        if (local_best_si.score is not None or
           local_best_si.is_model_sharpening()) and (
            best_si.score is None or local_best_si.score > best_si.score):
          best_si=local_best_si
          best_map_and_b=local_best_map_and_b
          if not best_si.is_model_sharpening() and \
              not best_si.is_half_map_sharpening():
            print("This is the current best score\n", file=out)
  
    if (best_si.score is not None )  and (
       not best_si.is_model_sharpening() )  and (not best_si.is_half_map_sharpening()):
      print("\nOverall best sharpening method: %s Score: %7.3f\n" %(
         best_si.sharpening_method,best_si.score), file=out)
      best_si.show_summary(out=out)
  
    if (not best_si.is_model_sharpening()) and \
         (not best_si.is_half_map_sharpening()) and null_si:
      if best_si.score>null_si.score:  # we improved them..
        print("Improved score with sharpening...", file=out)
      else:
        print("Did not improve score with sharpening...", file=out)
    if return_bsi:
      map_data=best_map_and_b.map_data
      map_data=set_mean_sd_of_map(map_data=map_data,
        target_mean=starting_mean,target_sd=starting_sd)
  
      box_sharpening_info_obj.map_data=map_data
      box_sharpening_info_obj.smoothed_box_mask_data=smoothed_box_mask_data
      box_sharpening_info_obj.original_box_map_data=original_box_map_data
      box_sharpening_info_obj.n_buffer=n_buffer
      box_sharpening_info_obj.crystal_symmetry=best_si.crystal_symmetry
      box_sharpening_info_obj.resolution=best_si.resolution
      box_sharpening_info_obj.d_min_ratio=best_si.d_min_ratio
      box_sharpening_info_obj.scale_max=best_si.scale_max
      box_sharpening_info_obj.smoothing_radius=best_si.smoothing_radius
      box_sharpening_info_obj.b_iso=best_map_and_b.final_b_iso
      box_sharpening_info_obj.starting_b_iso=best_map_and_b.starting_b_iso
      return box_sharpening_info_obj
  
    if original_box_sharpening_info_obj:
        # Put back original crystal_symmetry with original_box_sharpening_info_obj
        print("\nRestoring original symmetry to best sharpening info", file=out)
        best_si.update_with_box_sharpening_info(
          box_sharpening_info_obj=original_box_sharpening_info_obj)
        print("(%7.3f, %7.3f, %7.3f, %7.3f, %7.3f, %7.3f) "%(tuple(
          best_si.crystal_symmetry.unit_cell().parameters())), file=out)
        # and set tracking data with result
    return best_si
  
  def run_sharpen_and_score(f_array=None,
    phases=None,
    local_si=None,
    crystal_symmetry=None,
    original_b_iso=None,
    m=None,
    return_bsi=None,
    id=None,
    out=sys.stdout):
  
          local_map_and_b=apply_sharpening(
              f_array=f_array,phases=phases,
              sharpening_info_obj=local_si,
              crystal_symmetry=crystal_symmetry,
              out=null_out())
          local_si=score_map(map_data=local_map_and_b.map_data,
            sharpening_info_obj=local_si,
            out=null_out())
          # Record b_iso values
          if not local_map_and_b.starting_b_iso:
            local_map_and_b.starting_b_iso=original_b_iso
          if not local_map_and_b.final_b_iso:
            local_map_and_b.final_b_iso=local_si.b_iso
  
          # This is printout below here ===============
  
          if m=='resolution_dependent':
            text=\
             "\nb[0]   b[1]   b[2]   SA   Kurtosis   sa_ratio  Normalized regions"
            text+="\n"+\
              "\nB-sharpen   B-iso   k_sharpen   SA   "+\
               "Kurtosis  sa_ratio  Normalized regions"
            text+="\n"+" %6.2f  %6.2f  %6.2f  " %(
                local_si.resolution_dependent_b[0],
                local_si.resolution_dependent_b[1],
                local_si.resolution_dependent_b[2]) +\
              "  %7.3f  %7.3f  " %(
                  local_si.adjusted_sa,local_si.kurtosis)+\
              " %7.3f  %7.3f" %(
               local_si.sa_ratio,local_si.normalized_regions)
          elif local_si.b_sharpen is not None and local_si.b_iso is not None and\
             local_si.k_sharpen is not None and local_si.kurtosis is not None \
             and local_si.adjusted_sa is not None:
            text=\
             " %6.1f     %6.1f  %5s   %7.3f  %7.3f" %(
              local_si.b_sharpen,local_si.b_iso,
               local_si.k_sharpen,local_si.adjusted_sa,local_si.kurtosis) + \
              "  %7.3f         %7.3f" %(
               local_si.sa_ratio,local_si.normalized_regions)
          else:
            text=""
  
          if return_bsi:
            r=group_args(
              local_si=local_si,
              local_map_and_b=local_map_and_b,
              text=text,
              id=id)
          else:
            r=group_args(
              local_si=local_si,
              local_map_and_b=None,
              text=text,
              id=id)
          return r
  
  def effective_b_iso(map_data=None,tracking_data=None,
        box_sharpening_info_obj=None,
        crystal_symmetry=None,
        resolution=None,
        remove_aniso=None,
        d_min_ratio=None,
        scale_max=None,
        out=sys.stdout):
      if not crystal_symmetry:
        if box_sharpening_info_obj:
          crystal_symmetry=box_sharpening_info_obj.crystal_symmetry
        else:
          crystal_symmetry=tracking_data.crystal_symmetry
      if resolution:
         d_min=resolution
      else:
         d_min=tracking_data.params.crystal_info.resolution
  
      if not d_min_ratio:
         d_min_ratio=tracking_data.params.map_modification.d_min_ratio
  
      map_coeffs,map_coeffs_ra=get_f_phases_from_map(map_data=map_data,
         crystal_symmetry=crystal_symmetry,
         d_min=d_min,
         d_min_ratio=d_min_ratio,
         scale_max=scale_max,
         remove_aniso=remove_aniso,
         return_as_map_coeffs=True,
         out=out)
  
      f_array,phases=map_coeffs_as_fp_phi(map_coeffs)
      b_iso=map_coeffs_ra.b_iso
      if b_iso is not None:
        print("Effective B-iso = %7.2f\n" %(b_iso), file=out)
      else:
        print("Effective B-iso not determined\n", file=out)
      return map_coeffs_ra,map_coeffs,f_array,phases
  
  def update_tracking_data_with_sharpening(map_data=None,tracking_data=None,
         si=None,out=sys.stdout):
  
      # Set shifted_map_info if map_data is new
      if tracking_data.params.output_files.shifted_sharpened_map_file:
        shifted_sharpened_map_file=os.path.join(
            tracking_data.params.output_files.output_directory,
            tracking_data.params.output_files.shifted_sharpened_map_file)
      else:
        shifted_sharpened_map_file=None
      from cctbx.maptbx.segment_and_split_map import write_ccp4_map
      if shifted_sharpened_map_file:
        write_ccp4_map(tracking_data.crystal_symmetry,
            shifted_sharpened_map_file,map_data)
        print("Wrote shifted, sharpened map to %s" %(
            shifted_sharpened_map_file), file=out)
      tracking_data.set_shifted_map_info(file_name=
            shifted_sharpened_map_file,
            crystal_symmetry=tracking_data.crystal_symmetry,
            origin=map_data.origin(),
            all=map_data.all(),
            b_sharpen=None)
  
  
  def get_high_points_from_map(
       map_data=None,
       boundary_radius=5.,
       unit_cell=None,
       out=sys.stdout):
      max_in_map_data=map_data.as_1d().min_max_mean().max
      for cutoff in [0.99,0.98,0.95,0.90,0.50]:
        high_points_mask=(map_data>= cutoff*max_in_map_data)
        sda=map_data.as_1d().min_max_mean().max
        for nth_point in [4,2,1]:
          sites_cart=get_marked_points_cart(mask_data=high_points_mask,
            unit_cell=unit_cell,every_nth_point=nth_point,
            boundary_radius=boundary_radius)
          if sites_cart.size()>0: break
        if sites_cart.size()>0: break
      assert sites_cart.size()>0
      del high_points_mask
      sites_cart=sites_cart[:1]
      xyz_frac=unit_cell.fractionalize(sites_cart[0])
      value=map_data.value_at_closest_grid_point(xyz_frac)
      print("High point in map at (%7.2f %7.2f %7.2f) with value of %7.2f " %(
          sites_cart[0][0],sites_cart[0][1],sites_cart[0][2],value), file=out)
      return sites_cart
  
  def get_one_au(tracking_data=None,
      sites_cart=None,
      ncs_obj=None,
      map_data=None,
      starting_mask=None,
      radius=None,
      every_nth_point=None,
      removed_ncs=None,
      out=sys.stdout):
    unit_cell=tracking_data.crystal_symmetry.unit_cell()
  
    if removed_ncs: # take everything left
      mm=map_data.as_1d().min_max_mean()
      mask_threshold=mm.min+max(0.00001,0.0001*(mm.mean-mm.min)) # just above min
    else:
      mask_threshold=tracking_data.params.segmentation.mask_threshold
  
    every_nth_point=tracking_data.params.segmentation.grid_spacing_for_au
    radius=tracking_data.params.segmentation.radius
    if not radius:
      radius=set_radius(unit_cell=unit_cell,map_data=map_data,
       every_nth_point=every_nth_point)
      tracking_data.params.segmentation.radius=radius
    print("\nRadius for AU identification: %7.2f A" %(radius), file=out)
  
    overall_mask,max_in_sd_map,sd_map=get_overall_mask(map_data=map_data,
      mask_threshold=mask_threshold,
      crystal_symmetry=tracking_data.crystal_symmetry,
      resolution=tracking_data.params.crystal_info.resolution,
      solvent_fraction=tracking_data.solvent_fraction,
      radius=radius,
      out=out)
  
    if starting_mask:
      print("Points in starting mask:",starting_mask.count(True), file=out)
      print("Points in overall mask:",overall_mask.count(True), file=out)
      print("Points in both:",(starting_mask & overall_mask).count(True), file=out)
      if tracking_data.params.crystal_info.is_crystal:
        # take starting mask as overall...
        overall_mask= starting_mask
      else: # usual
        # make sure overall mask is at least as big..
        overall_mask=(overall_mask | starting_mask)
      print("New size of overall mask: ",overall_mask.count(True), file=out)
    else:
      if not sites_cart: # pick top of map
        sites_cart=get_high_points_from_map(
          boundary_radius=radius,
          map_data=sd_map,
          unit_cell=unit_cell,out=out)
  
      starting_mask=mask_from_sites_and_map( # starting au mask
        map_data=sd_map,unit_cell=unit_cell,
        sites_cart=sites_cart,radius=radius,
        overall_mask=overall_mask)
  
    del sd_map
  
    au_mask,ncs_mask=get_ncs_mask(
      map_data=map_data,unit_cell=unit_cell,ncs_object=ncs_obj,
      starting_mask=starting_mask,
      radius=radius,
      overall_mask=overall_mask,
      every_nth_point=every_nth_point)
  
    print("Points in au: %d  in ncs: %d  (total %7.1f%%)   both: %d Not marked: %d" %(
       au_mask.count(True),ncs_mask.count(True),
       100.*float(au_mask.count(True)+ncs_mask.count(True))/au_mask.size(),
       (au_mask & ncs_mask).count(True),
       au_mask.size()-au_mask.count(True)-ncs_mask.count(True),), file=out)
  
    return au_mask
  
  def set_up_sharpening(si=None,map_data=None,out=sys.stdout):
           print("\nCarrying out specified sharpening/blurring of map", file=out)
           check_si=si  # just use input information
           check_si.show_summary(out=out)
           if check_si.is_target_b_iso_to_d_cut():
             check_si.b_iso=check_si.get_target_b_iso()
             check_si.b_sharpen=None
             print("Setting target b_iso of %7.1f " %(check_si.b_iso), file=out)
           if check_si.b_sharpen is None and check_si.b_iso is not None:
             # need to figure out b_sharpen
             print("\nGetting b_iso of map", file=out)
             b_iso=check_si.get_effective_b_iso(map_data=map_data,out=out)
             check_si.b_sharpen=b_iso-check_si.b_iso # sharpen is what to
             print("Value of b_sharpen to obtain b_iso of %s is %5.2f" %(
               check_si.b_iso,check_si.b_sharpen), file=out)
           elif check_si.b_sharpen is not None:
             print("Sharpening b_sharpen will be %s" %(check_si.b_sharpen), file=out)
           elif check_si.resolution_dependent_b:
             print("Resolution-dependent b_sharpening values:" +\
                "b0: %7.2f  b1: %7.2f  b2: %7.2f " %(
               tuple(check_si.resolution_dependent_b)), file=out)
           elif check_si.target_scale_factors:
             print("Model sharpening scale values:", file=out)
             for x in check_si.target_scale_factors: print(x, end=' ', file=out)
             print(file=out)
           return check_si
  
  def run(args,
       params=None,
       map_data=None,
       crystal_symmetry=None,
       sequence=None,
       half_map_data_list=None,
       ncs_obj=None,
       tracking_data=None,
       target_scattered_points=None,
       is_iteration=False,
       pdb_hierarchy=None,
       target_xyz=None,
       target_hierarchy=None,
       sharpening_target_pdb_inp=None,
       out=sys.stdout):
  
    if is_iteration:
      print("\nIteration tracking data:", file=out)
      tracking_data.show_summary(out=out)
    else:
      # get the parameters and map_data (sharpened, magnified, shifted...)
      params,map_data,half_map_data_list,pdb_hierarchy,tracking_data,\
          shifted_ncs_object=get_params( #
         args,map_data=map_data,crystal_symmetry=crystal_symmetry,
         half_map_data_list=half_map_data_list,
         ncs_object=ncs_obj,
         sequence=sequence,
         sharpening_target_pdb_inp=sharpening_target_pdb_inp,out=out)
      if params.control.shift_only:
        return map_data,ncs_obj,tracking_data
      elif params.control.check_ncs or \
          params.control.sharpen_only:
        return None,None,tracking_data
  
      if params.input_files.pdb_to_restore:
        restore_pdb(params,tracking_data=tracking_data,out=out)
        return None,None,tracking_data
      # read and write the ncs (Normally point-group NCS)
      ncs_obj,tracking_data=get_ncs(params=params,tracking_data=tracking_data,
         ncs_object=shifted_ncs_object,
         out=out)
  
      if params.input_files.target_ncs_au_file: # read in target
        import iotbx.pdb
        target_hierarchy=iotbx.pdb.input(
           file_name=params.input_files.target_ncs_au_file).construct_hierarchy()
  
      print("\nShifting model based on origin shift (if any)", file=out)
      print("Coordinate shift is (%7.2f,%7.2f,%7.2f)" %(
          tuple(tracking_data.origin_shift)), file=out)
      if not map_data:
         raise Sorry("Need map data for segment_and_split_map")
  
      if params.output_files.shifted_map_file:
          shifted_map_file=os.path.join(
            tracking_data.params.output_files.output_directory,
            params.output_files.shifted_map_file)
      else:
        shifted_map_file=None
      if params.output_files.shifted_ncs_file:
          shifted_ncs_file=os.path.join(
            tracking_data.params.output_files.output_directory,
            params.output_files.shifted_ncs_file)
      else:
        shifted_ncs_file=None
      if params.output_files.shifted_ncs_file:
          shifted_pdb_file=os.path.join(
            tracking_data.params.output_files.output_directory,
            params.output_files.shifted_pdb_file)
      else:
        shifted_pdb_file=None
      ncs_obj,pdb_hierarchy,target_hierarchy,\
        tracking_data,sharpening_target_pdb_inp=apply_origin_shift(
          shifted_map_file=shifted_map_file,
          shifted_pdb_file=shifted_pdb_file,
          shifted_ncs_file=shifted_ncs_file,
          origin_shift=tracking_data.origin_shift,
          shifted_ncs_object=shifted_ncs_object,
          pdb_hierarchy=pdb_hierarchy,
          target_hierarchy=target_hierarchy,
          map_data=map_data,
          tracking_data=tracking_data,
          sharpening_target_pdb_inp=sharpening_target_pdb_inp,
          out=out)
  
      if target_hierarchy:
        target_xyz=target_hierarchy.atoms().extract_xyz()
        del target_hierarchy
  
      # We can use params.input_files.target_ncs_au_file here to define ncs au
      if target_xyz and not target_scattered_points:
         target_scattered_points=flex.vec3_double()
         target_scattered_points.append(target_xyz.mean())
  
      # get the chain types and therefore (using ncs_copies) volume fraction
      tracking_data=get_solvent_fraction(params,
        ncs_object=ncs_obj,tracking_data=tracking_data,out=out)
  
      # Done with getting params and maps
      # Summarize after any sharpening
      tracking_data.show_summary(out=out)
  
    original_ncs_obj=ncs_obj # in case we need it later...
    original_input_ncs_info=tracking_data.input_ncs_info
    removed_ncs=False
  
    n_residues=tracking_data.n_residues
    ncs_copies=tracking_data.input_ncs_info.number_of_operators
    if (not tracking_data.solvent_fraction) and \
        params.crystal_info.molecular_mass:
      tracking_data.solvent_fraction=get_solvent_fraction_from_molecular_mass(
          crystal_symmetry=tracking_data.crystal_symmetry,
          molecular_mass=params.crystal_info.molecular_mass,
          out=out)
    if tracking_data.solvent_fraction:
      solvent_fraction=tracking_data.solvent_fraction
    else:
      raise Sorry("Need solvent fraction or molecular mass or sequence file")
  
    # Now usual method, using our new map...should duplicate best result above
    for itry in range(2):
      # get connectivity  (conn=connectivity_object.result)
      b_vs_region=b_vs_region_info()
      si=sharpening_info(tracking_data=tracking_data)
      co,sorted_by_volume,min_b,max_b,unique_expected_regions,best_score,\
         new_threshold,starting_density_threshold=\
           get_connectivity(
             b_vs_region=b_vs_region,
             map_data=map_data,
             iterate_with_remainder=params.segmentation.iterate_with_remainder,
             n_residues=n_residues,
             ncs_copies=ncs_copies,
             solvent_fraction=solvent_fraction,
             fraction_occupied=si.fraction_occupied,
             min_volume=si.min_volume,
             min_ratio=si.min_ratio,
             wrapping=si.wrapping,
             residues_per_region=si.residues_per_region,
             max_ratio_to_target=si.max_ratio_to_target,
             min_ratio_to_target=si.min_ratio_to_target,
             min_ratio_of_ncs_copy_to_first=si.min_ratio_of_ncs_copy_to_first,
             starting_density_threshold=si.starting_density_threshold,
             density_threshold=si.density_threshold,
             crystal_symmetry=si.crystal_symmetry,
             chain_type=si.chain_type,
             verbose=si.verbose,
             out=out)
      params.segmentation.starting_density_threshold=starting_density_threshold # have to set tracking data as we are passing that above
      tracking_data.params.segmentation.starting_density_threshold=starting_density_threshold # have to set tracking data as we are passing that above
      if new_threshold:
        print("\nNew threshold is %7.2f" %(new_threshold), file=out)
      if co is None: # no luck
        return None,None,tracking_data
  
      # Check to see which regions are in more than one au of the NCS
      #   and set them aside.  Group ncs-related regions together
  
      ncs_group_obj,tracking_data,equiv_dict_ncs_copy=identify_ncs_regions(
         params,sorted_by_volume=sorted_by_volume,
         co=co,
         min_b=min_b,
         max_b=max_b,
         ncs_obj=ncs_obj,
         tracking_data=tracking_data,
         out=out)
      if ncs_group_obj and ncs_group_obj.ncs_group_list: # ok
        break
      elif ncs_obj and itry==0 and not is_iteration:# try again
        print("No NCS groups identified on first try...taking entire NCS AU.", file=out)
        # Identify ncs au
        au_mask=get_one_au(tracking_data=tracking_data,
          ncs_obj=ncs_obj,
          map_data=map_data,out=out)
        s=(au_mask==False)
        min_in_map=map_data.as_1d().min_max_mean().min
        map_data.set_selected(s,min_in_map)  # mask out all but au
        from mmtbx.ncs.ncs import ncs
        ncs_obj=ncs()
        ncs_obj.set_unit_ncs()
        tracking_data.set_ncs_obj(ncs_obj=None)
        tracking_data.update_ncs_info(number_of_operators=1)
        if n_residues:
          n_residues=n_residues/ncs_copies
        solvent_fraction=max(0.001,min(0.999,
         1-((1-solvent_fraction)/ncs_copies)))
        ncs_copies=1
        params.segmentation.require_complete=False
        params.segmentation.iterate_with_remainder=False # so we do not iterate
        removed_ncs=True
        # Run again
      else: # tried twice, give up
        return None,None,tracking_data
  
    # Choose one region or group of regions from each ncs_group in the list
    #  Optimize the closeness of centers
  
    # Select group of regions that are close together and represent one au
  
    ncs_group_obj,scattered_points=\
       select_regions_in_au(
       params,
       ncs_group_obj=ncs_group_obj,
       equiv_dict_ncs_copy=equiv_dict_ncs_copy,
       tracking_data=tracking_data,
       target_scattered_points=target_scattered_points,
       unique_expected_regions=unique_expected_regions,
       out=out)
  
    # write out mask and map for all the selected regions...
  
    # Iterate if desired
    if params.segmentation.iterate_with_remainder and \
        ncs_group_obj.selected_regions:
  
      print("\nCreating remaining mask and map", file=out)
      map_data_remaining=create_remaining_mask_and_map(params,
        ncs_group_obj=ncs_group_obj,
        map_data=map_data,
        crystal_symmetry=tracking_data.crystal_symmetry,
        out=out)
  
      remainder_ncs_group_obj=iterate_search(params,
        map_data=map_data,
        map_data_remaining=map_data_remaining,
        ncs_obj=ncs_obj,
        ncs_group_obj=ncs_group_obj,
        scattered_points=scattered_points,
        tracking_data=tracking_data,
        out=out)
    else:
      remainder_ncs_group_obj=None
  
    # collect all NCS ops that are needed to relate all the regions
    #  that are used
    ncs_ops_used=ncs_group_obj.ncs_ops_used
    if remainder_ncs_group_obj and remainder_ncs_group_obj.ncs_ops_used:
      for x in remainder_ncs_group_obj.ncs_ops_used:
        if not x in ncs_ops_used: ncs_ops_used.append(x)
    if ncs_ops_used:
      ncs_ops_used.sort()
      print("Final NCS ops used: ",ncs_ops_used, file=out)
  
    # Save the used NCS ops
    ncs_used_obj=ncs_group_obj.ncs_obj.deep_copy(ops_to_keep=ncs_ops_used)
    if params.output_files.shifted_used_ncs_file:
      shifted_used_ncs_file=os.path.join(
        tracking_data.params.output_files.output_directory,
        params.output_files.shifted_used_ncs_file)
      ncs_used_obj.format_all_for_group_specification(
           file_name=shifted_used_ncs_file)
      tracking_data.set_shifted_used_ncs_info(file_name=shifted_used_ncs_file,
        number_of_operators=ncs_used_obj.max_operators(),
        is_helical_symmetry=tracking_data.input_ncs_info.is_helical_symmetry)
      tracking_data.shifted_used_ncs_info.show_summary(out=out)
  
    # Write out final maps and dummy atom files
    if params.output_files.write_output_maps:
      print("\nWriting output maps", file=out)
    else:
      print("\nSetting up but not writing output maps", file=out)
    map_files_written=write_output_files(params,
        tracking_data=tracking_data,
        map_data=map_data,
        half_map_data_list=half_map_data_list,
        ncs_group_obj=ncs_group_obj,
        remainder_ncs_group_obj=remainder_ncs_group_obj,
        pdb_hierarchy=pdb_hierarchy,
        removed_ncs=removed_ncs,
        out=out)
    ncs_group_obj.set_map_files_written(map_files_written)
  
    # Restore ncs info if we removed it
    if removed_ncs:
      print("\nRestoring original NCS info to tracking_data", file=out)
      tracking_data.input_ncs_info=original_input_ncs_info
  
  
    if params.output_files.output_info_file and ncs_group_obj:
      write_info_file(params=params,tracking_data=tracking_data,out=out)
  
    return ncs_group_obj,remainder_ncs_group_obj,tracking_data
  
  
  if __name__=="__main__":
    run(args=sys.argv[1:])
  