Exactly how do backslashes work within backticks?

Question

From the Bash FAQ:

Backslashes (\\) inside backticks are handled in a non-obvious manner:

 $ echo "`echo \\\\a`" "$(echo \\         


        

Answer 1

Did some more research to find the reference and rule of what is happening. From the GNU Bash Reference Manual it states

When the old-style backquote form of substitution is used, backslash retains its literal meaning except when followed by ‘$’, ‘`’, or ‘\’. The first backquote not preceded by a backslash terminates the command substitution. When using the $(command) form, all characters between the parentheses make up the command; none are treated specially.

In other words \, \$, and ` inside of `` are processed by the CLI parser before the command substitution. Everything else is passed to the command substitution for processing.

Let's step through each example from the question. After the # I put how the command substitution was processed by the CLI parser before `` or $() is executed.

Your first example explained.

$ echo "`echo \\a`"   # echo \a
 a 
$ echo "$(echo \\a)"  # echo \\a
 \a

Your second example explained:

$ echo "`echo \\\\a`"   # echo \\a
 \a 
$ echo "$(echo \\\\a)"  # echo \\\\a
 \\a

Your third example:

a=xx
$ echo "`echo $a`"    # echo xx 
xx
$ echo "`echo \$a`"   # echo $a
xx
echo "`echo \\$a`"    # echo \$a
$a

Your third example using $()

$ echo "$(echo $a)"     # echo $a
xx
$ echo "$(echo \$a)"    # echo \$a
$a
$ echo "$(echo \\$a)"   # echo \\$a
\xx

Answer 2

The logic is quite simple as such. So we look at bash source code (4.4) itself

subst.c:9273

case '`': /* Backquoted command substitution. */
{
    t_index = sindex++;

    temp = string_extract(string, &sindex, "`", SX_REQMATCH);
    /* The test of sindex against t_index is to allow bare instances of
        ` to pass through, for backwards compatibility. */
    if (temp == &extract_string_error || temp == &extract_string_fatal)
    {
    if (sindex - 1 == t_index)
    {
        sindex = t_index;
        goto add_character;
    }
    last_command_exit_value = EXECUTION_FAILURE;
    report_error(_("bad substitution: no closing \"`\" in %s"), string + t_index);
    free(string);
    free(istring);
    return ((temp == &extract_string_error) ? &expand_word_error
                                            : &expand_word_fatal);
    }

    if (expanded_something)
    *expanded_something = 1;

    if (word->flags & W_NOCOMSUB)
    /* sindex + 1 because string[sindex] == '`' */
    temp1 = substring(string, t_index, sindex + 1);
    else
    {
    de_backslash(temp);
    tword = command_substitute(temp, quoted);
    temp1 = tword ? tword->word : (char *)NULL;
    if (tword)
        dispose_word_desc(tword);
    }
    FREE(temp);
    temp = temp1;
    goto dollar_add_string;
}

As you can see calls a function de_backslash(temp); on the string which updates the string in c. The code the same function is below

subst.c:1607

/* Remove backslashes which are quoting backquotes from STRING.  Modifies
   STRING, and returns a pointer to it. */
char *
    de_backslash(string) char *string;
{
  register size_t slen;
  register int i, j, prev_i;
  DECLARE_MBSTATE;

  slen = strlen(string);
  i = j = 0;

  /* Loop copying string[i] to string[j], i >= j. */
  while (i < slen)
  {
    if (string[i] == '\\' && (string[i + 1] == '`' || string[i + 1] == '\\' ||
                              string[i + 1] == '$'))
      i++;
    prev_i = i;
    ADVANCE_CHAR(string, slen, i);
    if (j < prev_i)
      do
        string[j++] = string[prev_i++];
      while (prev_i < i);
    else
      j = i;
  }
  string[j] = '\0';

  return (string);
}

The above just does simple thing if there is \ character and the next character is \ or backtick or $, then skip this \ character and copy the next character

So if convert it to python for simplicity

text = r"\\\\$a"

slen = len(text)
i = 0
j = 0
data = ""
while i < slen:
    if (text[i] == '\\' and (text[i + 1] == '`' or text[i + 1] == '\\' or
                             text[i + 1] == '$')):
        i += 1
    data += text[i]
    i += 1

print(data)

The output of the same is \\$a. And now lets test the same in bash

$ a=xxx

$ echo "$(echo \\$a)"
\xxx

$ echo "`echo \\\\$a`"
\xxx