If you need a try/catch block anyway then the using statement is not buying you much. Just ditch it and do this instead:
StreamWriter sw = null;
try
{
sw = File.AppendText(filePath);
sw.WriteLine(message);
}
catch(Exception)
{
}
finally
{
if (sw != null)
sw.Dispose();
}