<xsl:variable as="xs:string" name="within-braces">
     [{]      <!--* opening bace *-->
     (        <!--* $1 *-->
       [^{}]+ <!--* not containing a brace *-->
     )        <!--* end $1 *-->
     [}]      <!--* closing brace *-->
</xsl:variable>

and then fn:replace($input, $within-braces, 'x'), or more likely,
   <xsl:analyze-string regex="$within-braces" flags="x" . .. . >

Correction:
regex="{$within-braces}"