Let\'s take this map, where \'#\' illustrates a taken square and \'.\' illustrates a free square:
1 . # # # . . 2 . # . . # . 3 # . . . . # 4 . # # # . . 5 . . . . .
I would start from each neighbor of the picked square, and try to 'escape' to the boundary of the grid. Meanwhile, mark the path followed by 'X'. If you can escape: undo every 'X'. If you cannot escape, replace every 'X' by '#'. I made an example in Java, as shown below.
int W, H;
char[][] input;
final int[][] directions = {{-1, 0}, {1, 0}, {0, -1}, {0, 1}};
public void handle(int x, int y) {
// try each neihgbor
for (int[] d : directions) {
if (canEscape(input, x, y)) {
// if we can escape, the path found shouldn't be filled
// so replace the Xes by '.';
handleXes(input, false);
} else {
// if we cannot escape, this is a closed shape, so
// fill with '#'
handleXes(input, true);
}
// note that this can be written more concisely as
// handleXes(input, !canEscape(input, x, y));
}
}
public boolean canEscape(char[][] grid, int x, int y) {
if (isEscape(grid, x, y))
return true
if (isValid(grid, x, y)) {
// mark as visited
grid[x][y] = 'X';
// try each neighbor
for (int[] d : directions) {
if (canEscape(grid, x+d[0], y+d[1]))
return true;
}
}
return false;
}
public boolean isValid(char[][] grid, int x, int y) {
return 0 <= x && x < W && 0 <= y && y < H && grid[x][y] == '.';
}
public boolean isEscape(char[][] grid, int x, int y) {
return (0 == x || x == W-1 || 0 == y || y == H-1) && grid[x][y] == '.';
}
public void handleXes(char[][] grid, boolean fill) {
for (int x = 0; x < W; x++)
for (int y = 0; y < H; y++)
if (grid[x][y] == 'X')
grid[x][y] = fill ? '#' : '.';
}