How to select only visible vertices using python API?
First of all, as for the select_all behavior, that is by design.
Calling bpy.ops.object.select_all(action='SELECT') is the same thing as pressing the A key, which selects ALL items, even ones outside of the view frustum, even ones occluded and not visible.
To select visible vertices in python similar to border or circle select honoring bpy.types.SpaceView3D.use_occlude_geometry and bpy.ops.mesh.select_mode:
import bpy

def getView3dAreaAndRegion(context):
  for area in context.screen.areas:
      if area.type == &quot;VIEW_3D&quot;:
        for region in area.regions:
          if region.type == &quot;WINDOW&quot;:
            return area, region

def select_border(context, view3dAreaAndRegion=None, extend=True):
  if not view3dAreaAndRegion:
    view3dAreaAndRegion = getView3dAreaAndRegion(context)
  view3dArea, view3dRegion = view3dAreaAndRegion
  override = context.copy()
  override['area'] = view3dArea
  override['region'] = view3dRegion
  bpy.ops.view3d.select_border(override,
                               gesture_mode=3,
                               xmin=0,
                               xmax=view3dArea.width,
                               ymin=0,
                               ymax=view3dArea.height,
                               extend=extend)
  return view3dAreaAndRegion

Then to use this, just add after the function definitions:
select_border(bpy.context)

Blender &gt;2.8
def getView3dAreaAndRegion(context):
  for area in context.screen.areas: 
      if area.type == &quot;VIEW_3D&quot;:    
        for region in area.regions:
          if region.type == &quot;WINDOW&quot;:
            print(&quot;Found WINDOW&quot;) 
            return area, region

def select_border(context, view3dAreaAndRegion=None, extend=True):
  if not view3dAreaAndRegion:
    view3dAreaAndRegion = getView3dAreaAndRegion(context)
    print(view3dAreaAndRegion)
  view3dArea, view3dRegion = view3dAreaAndRegion
  override = context.copy()
  override['area'] = view3dArea
  override['region'] = view3dRegion
  bpy.ops.view3d.select_box(override,xmin=0,xmax=view3dArea.width,ymin=0,ymax=view3dArea.height,mode='ADD')

and then, with a selected object in Edit mode, run
select_border(bpy.context)

Very important: This code needs to be run in a window that has a 3D Editor view in it. So if you're running the code editor in a separate detached window, it won't work !