Lock a row if it exists, create new record otherwise
If LockedBy is null when not locked then...

Using select @CurrentLockedBy = isnull(LockedBy,0) lets us find out which of the three states the record is in. If it is null, then the record doesn't exist. If it is 0 then it exists and is unlocked. Otherwise it exists and is locked by someone (possibly the caller).

I would use something like this: 

create procedure dbo.docs_upsert (
    @docid int output
  , @cols varchar(max)
  , @LockedBy int
  , @CallerUserId int
  ) as
begin
  set nocount on;
  set xact_abort on;
  declare @CurrentLockedBy int;
  begin try;
    begin tran;
      select @CurrentLockedBy = isnull(LockedBy,0)
        from docs d with (updlock, holdlock) -- lock for update/insert
        where d.id = @docid; 
    /*
     If @CurrentLockedBy &gt;0 then Locked by someone
     If @CurrentLockedBy = 0, exists and not locked. 
     If @CurrentLockedBy is null, row doesn't exist.
     */
    /* Locked by another user*/
    if isnull(@CurrentLockedBy,@CallerUserId)!=@CallerUserId
    begin;
      return -1;
    end;
    /* Exists, not Locked */
    if @CurrentLockedBy = 0
    begin;
      update docs set 
          cols = @cols
        , LockedBy = @LockedBy
        where id = @docid;
    end;
    else 
    /* Doesn't exist, insert */
    begin;
      insert into docs(cols)
        values(@cols);
        set @docid=scope_identity();
    end;
    commit tran;
  end try
  begin catch;
    if @@trancount &gt; 0 
      begin;
        rollback transaction;
        throw; -- or other error handling
      end;
  end catch;
end;