You are mistaken. Need AND. This code works correctly. It is possible to write so:

private bool buttonPressFlag = false;
private void Form1_MouseDown(object sender, MouseEventArgs e)
{
if (buttonPressFlag)
{
Color swap = ForeColor;
ForeColor = BackColor;
BackColor = swap;
}
if ((e.Button & MouseButtons.Left) == MouseButtons.Left ||
(e.Button & MouseButtons.Right) == MouseButtons.Right)
buttonPressFlag = true;
}

private void Form1_MouseUp(object sender, MouseEventArgs e)
{
if ((e.Button & MouseButtons.Left) == MouseButtons.Left ||
(e.Button & MouseButtons.Right) == MouseButtons.Right)
buttonPressFlag = false;
}

PS: Excuse me. At first time I has not understood your answer.

What you mean? I badly speak english.

private bool leftButtonPressFlag = false;
private bool rightButtonPressFlag = false;
private void Form1_MouseDown(object sender, MouseEventArgs e)
{
if (e.Button == MouseButtons.Left) leftButtonPressFlag = true;
if (e.Button == MouseButtons.Right) rightButtonPressFlag = true;
if (leftButtonPressFlag && rightButtonPressFlag)
{
Color swap = ForeColor;
ForeColor = BackColor;
BackColor = swap;
}
}

private void Form1_MouseUp(object sender, MouseEventArgs e)
{
if (e.Button == MouseButtons.Left) leftButtonPressFlag = false;
if (e.Button == MouseButtons.Right) rightButtonPressFlag = false;
}