I have an isometric grid system who\'s coordinates start from [0,0] in the left hand corner of t
Linear algebra is the answer. There are two coordinate systems of interest here: screen coordinates and isometric coordinates. Converting the corners of the selected region from screen coordinates to isometric coordinates will greatly help you.
Let theta be the angle between the x and y axes of the isometric coordinates, measured on the screen and unit be the pixel length of one step in the isometric coordinates. Then
var c = Math.cos(theta/2);
var s = Math.sin(theta/2);
var origin = [oX, oY]; //the pixel coordinates of (0, 0)
var unit = 20;
var iso2Screen = function(iso) {
var screenX = origin[0] + unit * (iso[0] * c + iso[1] * c);
var screenY = origin[1] + unit * (iso[0] * -s + iso[1] * s);
return [screenX, screenY];
}
Inverting this relationship, we get
var screen2Iso = function(screen) {
var isoX = ((screen[0] - origin[0]) / c - (screen[1] - origin[1]) / s) / unit;
var isoY = ((screen[0] - origin[0]) / c + (screen[1] - origin[1]) / s) / unit;
Now convert the screen coordinates of each corner of the selection box to isometric coordinates and get the minimum and maximum x and y.
var cornersScreen = ...//4-element array of 2-element arrays
var cornersIso = [];
for(var i = 0; i < 4; i++) {
cornersIso.push(screen2Iso(cornersScreen[i]));
}
var minX, maxX, minY, maxY;
minX = Math.min(cornersIso[0][0], cornersIso[1][0], cornersIso[2][0], cornersIso[3][0]);
maxX = Math.max(cornersIso[0][0], cornersIso[1][0], cornersIso[2][0], cornersIso[3][0]);
minY = Math.min(cornersIso[0][1], cornersIso[1][1], cornersIso[2][1], cornersIso[3][1]);
maxY = Math.max(cornersIso[0][1], cornersIso[1][1], cornersIso[2][1], cornersIso[3][1]);
All the selected isometric points lie inside of the isometric box [minX, maxX] x [minY, maxY], but not all of the points in that box are inside the selection.
You could do a lot of different things to weed out the points in that box that are not in the selection; I'd suggest iterating over integer values of the isometric x and y, converting the isometric coordinates to screen coordinates, and then testing to see if that screen coordinate lies within the selection box, to wit:
var sMinX, sMaxX, sMinY, sMaxY;
sMinX = Math.min(cornersScreen[0][0], cornersScreen[1][0], cornersScreen[2][0], cornersScreen[3][0]);
sMaxX = Math.max(cornersScreen[0][0], cornersScreen[1][0], cornersScreen[2][0], cornersScreen[3][0]);
sMinY = Math.min(cornersScreen[0][1], cornersScreen[1][1], cornersScreen[2][1], cornersScreen[3][1]);
sMaxY = Math.max(cornersScreen[0][1], cornersScreen[1][1], cornersScreen[2][1], cornersScreen[3][1]);
var selectedPoints = [];
for(var x = Math.floor(minX); x <= Math.ceil(maxX); x++) {
for(var y = Math.floor(minY); x <= Math.ceil(maxY); y++) {
var iso = [x,y];
var screen = iso2Screen(iso);
if(screen[0] >= sMinX && screen[0] <= sMaxX && screen[1] >= sMinY && screen[1] <= sMaxY) {
selectedPoints.push(iso);
}
}
}