问题
What is the best way to export data from multiple tables in MySQL. I'm basically working with product details. Say a product has 150 attributes of data. How can I export that in a single row and then export it to a flat file in CSV or tabdelimited format.
Getting error Too many tables; MySQL can only use 61 tables in a join
/**** Get Resultset *****/
$rs = mysql_query($sql);
/**** End of Get Resultset *****/
$objProfileHistory->addHistory($this->profile_id, "Loaded ". mysql_num_rows($rs)." records");
$this->runQuery($sql);
$this->exportToCSV();
/**
* getAttributeDetails
*/
function getAttributeDetails(){
global $dbObj, $profile;
$base_table = "catalog_product_entity";
$select = array();
$tables = array();
$i = 0;
$profile->showLog("Start fields mapping", "success");
if( is_array($this->attributes_in_db) && sizeof($this->attributes_in_db) > 0 ){
$arr = implode("','", $this->attributes_in_db);
$sql = "select attribute_id, attribute_code, backend_type, frontend_input
from eav_attribute
where attribute_code in ('".$arr."')
and entity_type_id =
(select entity_type_id
from eav_entity_type
where entity_type_code = 'catalog_product')";
$rs = $dbObj->customqry($sql);
if( $rs ){
while( $row = mysql_fetch_assoc( $rs ) ){
$backend_type = $row["backend_type"];
$attribut_code = $row["attribute_code"];
$attribute_id = $row["attribute_id"];
$frontend_input = $row["frontend_input"];
switch( $backend_type ){
case "text":
$where[] = $base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id;
$and[] = $base_table.".entity_id=".$base_table."_".$backend_type."".$i.".entity_id";
$select[] = $base_table."_".$backend_type."".$i.".value as ".$attribut_code;
$tables[] = $base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i;
break;
case "decimal":
$where[] = $base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id;
$and[] = $base_table.".entity_id=".$base_table."_".$backend_type."".$i.".entity_id";
$select[] = $base_table."_".$backend_type."".$i.".value as ".$attribut_code;
$tables[] = $base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i;
break;
case "static":
$where[] = $base_table."".$i.".entity_id=".$base_table.".entity_id";
$and[] = $base_table.".entity_id=".$base_table."".$i.".entity_id";
$select[] = $base_table."".$i.".".$attribut_code." as ".$attribut_code;
$tables[] = $base_table." as ".$base_table."".$i;
break;
case "int":
if( $attribut_code == "tax_class_id" && $frontend_input == "select" ){
$where[] = "tax_class{$i}.class_id=(select ".$base_table."_".$backend_type."".$i.".value from ".$base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i." where ".$base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id." and ".$base_table."_".$backend_type."".$i.".entity_id=".$base_table.".entity_id limit 1))";
$and[] = "";
$select[] = "tax_class{$i}.class_name as {$attribut_code}";
$tables[] = "tax_class as tax_class{$i}";
} else if( $frontend_input == "select" ){
$where[] = "eav_attribute_option_value{$i}.option_id=(select ".$base_table."_".$backend_type."".$i.".value from ".$base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i." where ".$base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id." and ".$base_table."_".$backend_type."".$i.".entity_id=".$base_table.".entity_id limit 1))";
$and[] = "";
$select[] = "eav_attribute_option_value{$i}.value as {$attribut_code}";
$tables[] = "eav_attribute_option_value as eav_attribute_option_value{$i}";
} else {
$where[] = $base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id;
$and[] = $base_table.".entity_id=".$base_table."_".$backend_type."".$i.".entity_id";
$select[] = $base_table."_".$backend_type."".$i.".value as ".$attribut_code;
$tables[] = $base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i;
}
break;
case "varchar":
$where[] = $base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id;
$and[] = $base_table.".entity_id=".$base_table."_".$backend_type."".$i.".entity_id";
$select[] = $base_table."_".$backend_type."".$i.".value as ".$attribut_code;
$tables[] = $base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i;
break;
case "datetime":
$where[] = $base_table."_".$backend_type."".$i.".attribute_id=".$attribute_id;
$and[] = $base_table.".entity_id=".$base_table."_".$backend_type."".$i.".entity_id";
$select[] = $base_table."_".$backend_type."".$i.".value as ".$attribut_code;
$tables[] = $base_table."_".$backend_type." as ".$base_table."_".$backend_type."".$i;
break;
}//switch
$i++;
}//while
$sql = "select ".implode(",", $select)." from ".$base_table;
for($i=0; $i < sizeof($select); $i++){
$sql .= " left join ". $tables[$i] . " on (".$where[$i];//." and ".$and[$i].")";
if( strlen($and[$i]) > 0 ){
$sql .= " and ".$and[$i].")";
}
}//for
$sql .= " group by {$base_table}.entity_id ";
}//if
//echo $sql; exit;
return $sql;
}
//echo $sql;
//echo "<pre>";print_r($tables);print_r($select);print_r($where);print_r($and);
}//end function
/**
* runQuery
*/
function runQuery( $sql ){
global $dbObj, $profile;
if( $sql != "" ){
$rs = $dbObj->customqry( $sql );
$profile->showLog("Loaded ". mysql_num_rows($rs) ." records", "success");
if( $rs ){
$i = 0;
while( $row = mysql_fetch_assoc( $rs ) ){
$cnt = sizeof($this->attributes_in_db);
for($j=0; $j < $cnt; $j++){
$db_key = $this->attributes_in_db[$j];
$file_key = $this->attributes_in_file[$j];
$this->export_data[$i][$db_key] = $row[$db_key];
}
$i++;
}//while
}
}//if
}//end function
/**
* exportToCSV
*/
function exportToCSV(){
global $smarty, $objProfileHistory, $profile;
//$newFileName = $smarty->root_dir."/export/".$this->filename; //file name that you want to create
$cnt = sizeof($this->var_array);
for($i=0; $i < $cnt; $i++){
extract($this->var_array[$i]);
}//for
if( $delimiter = "\t" ){
$delimiter = "\t";//$delimiter;
}
if( strlen($filename) < 1 ){
$filename = time().".csv";
}
// echo "<pre>";
// print_r($this->action_array);
// print_r($this->var_array);
// print_r($this->map_array);
// exit;
# add amazon headers
if( $this->action_array[0]['type'] == 'header' ){
// $template_type = $this->var_array[0]['template_type'];
// $version = $this->var_array[0]['version'];
// $status_message = $this->var_array[0]['status_message'];
$sStr = "TemplateType=".$template_type."{$delimiter}{$delimiter}Version=".$version."{$delimiter}{$delimiter}{$status_message}";
$sStr .= "� ��\n"; //to seprate every record
}
$export_path = $path;
$x_path = $profile->createDir( $export_path );
$newFileName = $x_path ."/". $filename;
$fpWrite = fopen($newFileName, "w"); // open file as writable
# create header
$cnt_header = sizeof($this->attributes_in_file);
for( $i=0; $i < $cnt_header; $i++){
$sStr .= $deli . $this->attributes_in_file[$i];
$deli = $delimiter;
}//for
$sStr .= "� ��\n"; //to seprate every record
# attach data
$cnt_row = sizeof($this->export_data);
for( $i=0; $i < $cnt_row; $i++ ){
$sStr .= $saperator;
$newdeli = "";
for($j=0; $j < $cnt_header; $j++){
$key = $this->attributes_in_db[$j];
$sku = $this->export_data[$i]["sku"];
回答1:
You're using an EAV design, and trying to re-construct a single row from a variable number of attributes. This points out one of the many landmines you'll encounter using the EAV design: there's a practical limit on the number of joins you can do in a single SQL query.
Especially in MySQL -- there's a hard limit, as you've found. But even in other RDBMS brands, there's an effective limit because the cost of joins is geometric with respect to the number of tables.
If you use EAV, don't try to re-construct a row in SQL as if you had a conventional database design. Instead, fetch the attributes as rows, sorted by the entity id. Then post-process them in your application code. This does mean that you can't dump the data in one step -- you have to write code to loop over the attribute rows, and reform each row of data before you can output it.
EAV is not a convenient database design. There are many expensive drawbacks to using it, and you've just hit one of them.
See http://www.simple-talk.com/opinion/opinion-pieces/bad-carma/ for a great story about how using EAV doomed one business.
And also see http://en.wikipedia.org/wiki/Inner-platform_effect because EAV is an example of this Anti-pattern.
I understand the need to support a dynamic set of attributes per product in a catalog. But EAV is going to kill your application. Here's what I do to support dynamic attributes:
Define a real column in the base table for each attribute that's common to all product types. Product name, price, quantity in stock, etc. Work hard to imagine the canonical product entity so you can include as many attributes as possible in this set.
Define one more column of type
TEXTfor all additional attributes of each given product type. Store in this column as Serialized LOB of the attributes, in whatever format suits you: XML, JSON, YAML, your own homemade DSL, etc.Treat this as a single column in your SQL queries. Any searching, sorting, or display you need to do based on these attributes requires you to fetch the whole
TEXTblob into your application deserialize it, and analyze the attributes using application code.
回答2:
If you have this many attributes, I expect that it is a sparse database, so you have a great deal of wasted space.
You may want to look at using an Entity-Attribute-Value database instead, if possible.
http://en.wikipedia.org/wiki/Entity-attribute-value_model
What this buys you is a way to refactor the database, but have it be more extensible, and reduce how many tables you need. You should be able to come down to 4-6 tables (2-3 entity tables with their attributes). It is a bit more difficult to create the queries as all the queries will be dynamic, but it will simplify your export, and the database maintenance should be simpler.
If you must use this schema you may want to create several triggers and then you can call the trigger, which is joining several tables, and then make your query, but you will take a huge performance hit.
UPDATE:
Since an EAV table is being used, and MySQL doesn't do a pivot function you may want to read the answer to this question: How to pivot a MySQL entity-attribute-value schema How to pivot a MySQL entity-attribute-value schema
回答3:
If you're using EAV and you want to export a large number of attributes at once, the best way is actually to use multiple temporary tables.
Each temporary table will have the same primary key column. Then join all of them and export into csv.
I don't know if I want to do a fully fleshed out example, but I will try to do an outline that will hopefully make things clearer.
1.) Get your list of attributes you want to export. You will use their attribute_ids in the join to your EAV attribute_values table.
2.) Split up the attributes so that you will not exceed the join limit. You need the original table, and 1 table per join, so you can have 60 attributes per table in this scheme.
3.) Create "flat" temporary tables for each group of attributes. It would go something like this.
CREATE TEMPORARY TABLE temp1
[(create_definition,...)]
SELECT t1.product_id, t1.sku, t2.color, GROUP_CONCAT(t3.sizes SEPARATOR ',') as sizes,
...
#( suppose the product has multiple sizes and you want them shown comma-separated in your export)
FROM products t1
LEFT JOIN eav_attribute_values t2 ON t1.product_id = t2.product_id AND t2.attribute_id = 55
LEFT JOIN eav_attribute_values t3 ON t1.product_id = t2.product_id AND t2.attribute_id = 76
... etc for up to 60 attributes
CREATE TEMPORARY TABLE temp2 ... # repeat for next 60 attributes
4.) Now you have temporary tables temp1, temp2, temp3, etc. They all share the same primary key (product_id and/or product_sku for example). Assuming you have less than 60 temporary tables (which would be absurd), you can now join all of those and create a single table.
In my system, I don't think I've exceeded 3 temporary tables and that is quite a lot.
CREATE TEMPORARY TABLE export_data
[(create_definition,...)]
SELECT t1.*, t2.*, t3.* FROM # though I would not actually use * here b/c it would cause repeated key fields. I would list out all the columns
temp1 t1 LEFT JOIN temp2 t2 ON t1.product_id = t2.product_id
LEFT JOIN temp3 t3 ON t1.product_id = t3.product_id # etc for more joins
5.) Export. Use MySQL's file export feature to create a CSV. Send it to the user with PHP.
I hope that helps.
Also note that the above process executes fairly quickly for me. The reason to use temporary tables is because they will be automatically dropped after use, and because multiple users can all run the same type of process without interfering with each other since temporary tables only exist for the user who created them.
回答4:
In Spring Boot this will occur if you are not using fetch type as LAZY. In this kind of situation one of your table has over 61 sub relationships EX - table names A,B,C and D
- A relate B
- B relate C
- C relate D
In this situation A has 3 sub relations. If you get a data from A you can access to the data in D based on relationships.
So use Fetch type as LAZY
来源：https://stackoverflow.com/questions/1684971/too-many-tables-mysql-can-only-use-61-tables-in-a-join