Unable to start hive metastore service or hive-shell after configuring mysql with hive

Question

I know this question asked already but those answers are not helping in any way.

I am spending more time to configure mysql with hive, Every time I encounter errors and

Answer 1

Problem was with hive-site.xml file and this configuration helped me to resolve the issue.

  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://localhost/metastore_db?createDatabaseIfNotExist=true</value>
    <description>metadata is stored in a MySQL server</description>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
    <description>MySQL JDBC driver class</description>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hiveuser</value>
    <description>user name for connecting to mysql server </description>
  </property>

  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>hivepassword</value>
    <description>password for connecting to mysql server </description>
  </property>
  <property>
    <name>javax.jdo.PersistenceManagerFactoryClass</name>
    <value>org.datanucleus.api.jdo.JDOPersistenceManagerFactory</value>
    <description>class implementing the jdo persistence</description>
  </property>

  <property>
    <name>javax.jdo.option.DetachAllOnCommit</name>
    <value>true</value>
    <description>detaches all objects from session so that they can be used after transaction is committed</description>
  </property>

  <property>
    <name>javax.jdo.option.NonTransactionalRead</name>
    <value>true</value>
    <description>reads outside of transactions</description>
  </property>
  <property>
    <name>javax.jdo.option.Multithreaded</name>
    <value>true</value>
    <description>Set this to true if multiple threads access metastore through JDO concurrently.</description>
  </property>

  <property>
    <name>datanucleus.validateTables</name>
    <value>false</value>
    <description>validates existing schema against code. turn this on if you want to verify existing schema </description>
  </property>

  <property>
    <name>datanucleus.validateColumns</name>
    <value>false</value>
    <description>validates existing schema against code. turn this on if you want to verify existing schema </description>
  </property>

  <property>
    <name>datanucleus.validateConstraints</name>
    <value>false</value>
    <description>validates existing schema against code. turn this on if you want to verify existing schema </description>
  </property>

  <property>
    <name>datanucleus.storeManagerType</name>
    <value>rdbms</value>
    <description>metadata store type</description>
  </property>

  <property>
    <name>datanucleus.autoCreateSchema</name>
    <value>false</value>
  </property>

  <property>
    <name>datanucleus.autoStartMechanismMode</name>
    <value>checked</value>
    <description>throw exception if metadata tables are incorrect</description>
  </property>

  <property>
    <name>datanucleus.autoStartMechanism</name>
    <value>SchemaTable</value>
  </property>

  <property>
    <name>datanucleus.fixedDatastore</name>
    <value>true</value>
  </property>

  <property>
    <name>datanucleus.transactionIsolation</name>
    <value>read-committed</value>
    <description>Default transaction isolation level for identity generation. </description>
  </property>

  <property>
    <name>datanucleus.cache.level2</name>
    <value>false</value>
    <description>Use a level 2 cache. Turn this off if metadata is changed independently of Hive metastore server</description>
  </property>

  <property>
    <name>datanucleus.cache.level2.type</name>
    <value>SOFT</value>
    <description>SOFT=soft reference based cache, WEAK=weak reference based cache.</description>
  </property>

  <property>
    <name>datanucleus.identifierFactory</name>
    <value>datanucleus1</value>
    <description>Name of the identifier factory to use when generating table/column names etc. 'datanucleus1' is used for backward  compatibility with DataNucleus v1</description>
  </property>


  <property>
    <name>datanucleus.plugin.pluginRegistryBundleCheck</name>
    <value>LOG</value>
    <description>Defines what happens when plugin bundles are found and are duplicated [EXCEPTION|LOG|NONE]</description>
  </property>

  <property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/user/hive/warehouse</value>
    <description>location of default database for the warehouse</description>
  </property>

  <property>
    <name>hive.metastore.execute.setugi</name>
    <value>false</value>
    <description>In unsecure mode, setting this property to true will cause the metastore to execute DFS operations using the client's reported user and group permissions. Note that this property must be set on both the client and server sides. Further  note that its best effort. If client sets its to true and server sets it to false, client setting will be ignored.</description>
  </property>

  <property>
    <name>hive.metastore.event.listeners</name>
    <value></value>
    <description>list of comma separated listeners for metastore events.</description>
  </property>

  <property>
    <name>hive.metastore.partition.inherit.table.properties</name>
    <value></value>
    <description>list of comma separated keys occurring in table properties which will get inherited to newly created partitions. *   implies all the keys will get inherited.</description>
  </property>

  <property>
    <name>hive.metadata.export.location</name>
    <value></value>
    <description>When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, it is the location to which the metadata will be exported. The default is an empty string, which results in the metadata being exported   to the current user's home directory on HDFS.</description>
  </property>

  <property>
    <name>hive.metadata.move.exported.metadata.to.trash</name>
    <value></value>
    <description>When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, this setting determines if the metadata that is exported will subsequently be moved to the user's trash directory alongside the   dropped table data. This ensures that the metadata will be cleaned up along with the dropped table data.</description>
  </property>

  <property>
    <name>hive.metastore.partition.name.whitelist.pattern</name>
    <value></value>
    <description>Partition names will be checked against this regex pattern and rejected if not matched.</description>
  </property>

  <property>
    <name>hive.metastore.disallow.incompatible.col.type.change</name>
    <value></value>
    <description>If true (default is false), ALTER TABLE operations which change the type of   a column (say STRING) to an incompatible type (say MAP&lt;STRING, STRING&gt;) are disallowed.    RCFile default SerDe (ColumnarSerDe) serializes the values in such a way that the  datatypes can be converted from string to any type. The map is also serialized as  a string, which can be read as a string as well. However, with any binary   serialization, this is not true. Blocking the ALTER TABLE prevents ClassCastExceptions  when subsequently trying to access old partitions.   Primitive types like INT, STRING, BIGINT, etc are compatible with each other and are   not blocked.  

  See HIVE-4409 for more details.
    </description>
  </property>

  <property>
    <name>hive.metastore.end.function.listeners</name>
    <value></value>
    <description>list of comma separated listeners for the end of metastore functions.</description>
  </property>

  <property>
    <name>hive.metastore.event.expiry.duration</name>
    <value>0</value>
    <description>Duration after which events expire from events table (in seconds)</description>
  </property>

  <property>
    <name>hive.metastore.event.clean.freq</name>
    <value>0</value>
    <description>Frequency at which timer task runs to purge expired events in metastore(in seconds).</description>
  </property>

  <property>
    <name>hive.metastore.connect.retries</name>
    <value>5</value>
    <description>Number of retries while opening a connection to metastore</description>
  </property>

  <property>
    <name>hive.metastore.failure.retries</name>
    <value>3</value>
    <description>Number of retries upon failure of Thrift metastore calls</description>
  </property>

  <property>
    <name>hive.metastore.client.connect.retry.delay</name>
    <value>1</value>
    <description>Number of seconds for the client to wait between consecutive connection attempts</description>
  </property>

  <property>
    <name>hive.metastore.client.socket.timeout</name>
    <value>20</value>
    <description>MetaStore Client socket timeout in seconds</description>
  </property>

  <property>
    <name>hive.metastore.rawstore.impl</name>
    <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
    <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store   and retrieval of raw metadata objects such as table, database</description>
  </property>

  <property>
    <name>hive.metastore.batch.retrieve.max</name>
    <value>300</value>
    <description>Maximum number of objects (tables/partitions) can be retrieved from metastore in one batch. The higher the number, the less the number of round trips is needed to the Hive metastore server, but it may also cause higher memory requirement at the   client side.</description>
  </property>

  <property>
    <name>hive.metastore.batch.retrieve.table.partition.max</name>
    <value>1000</value>
    <description>Maximum number of table partitions that metastore internally retrieves in one batch.</description>
  </property>
  <property>
    <name>hive.metastore.uris</name>
    <value>thrift://localhost:9083</value>
    <description>Hive metastore Thrift server</description>
  </property>
  <property>
    <name>mapred.reduce.tasks</name>
    <value>-1</value>
    <description>The default number of reduce tasks per job.  Typically set to a prime close to the number of available hosts.  Ignored when mapred.job.tracker is "local". Hadoop set this to 1 by default, whereas Hive uses -1 as its default value. By setting this property to -1, Hive will automatically figure out what should be the number of reducers.
    </description>
  </property>
  <property>
    <name>hive.cli.print.header</name>
    <value>false</value>
    <description>Whether to print the names of the columns in query output.</description>
  </property>

  <property>
    <name>hive.cli.print.current.db</name>
    <value>false</value>
    <description>Whether to include the current database in the Hive prompt.</description>
  </property>

  <property>
    <name>hive.cli.prompt</name>
    <value>hive</value>
    <description>Command line prompt configuration value. Other hiveconf can be used in this configuration value. Variable substitution will only be invoked at the Hive
    CLI startup.</description>
  </property>

  <property>
    <name>hive.test.mode</name>
    <value>false</value>
    <description>Whether Hive is running in test mode. If yes, it turns on sampling and prefixes the output tablename.</description>
  </property>

  <property>
    <name>hive.test.mode.prefix</name>
    <value>test_</value>
    <description>if Hive is running in test mode, prefixes the output table by this string</description>
  </property>


  <property>
    <name>hive.test.mode.samplefreq</name>
    <value>32</value>
    <description>if Hive is running in test mode and table is not bucketed, sampling frequency</description>
  </property>

  <property>
    <name>hive.test.mode.nosamplelist</name>
    <value></value>
    <description>if Hive is running in test mode, don't sample the above comma separated list of tables</description>
  </property>

  <property>
    <name>hive.metastore.uris</name>
    <value></value>
    <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description>
  </property>
</configuration>

Answer 2

            copy mysql sql driver in $HIVE_HOME/lib folder


            change hive-site.xml file properties under $HIVE_HOME/conf
            #########################################################

            <property>
            <name>javax.jdo.option.ConnectionDriverName</name>
            <value>com.mysql.jdbc.Driver</value>
            <description>Driver class name for a JDBC metastore</description>
            </property>



            <property>
                <name>javax.jdo.option.ConnectionURL</name>
                <value>jdbc:mysql://localhost:3306/hive_metastore_db?createDatabaseIfNotExist=true</value>
              </property>



            <property>
              <name>datanucleus.autoCreateSchema</name>
              <value>true</value>
            </property>

            <property>
              <name>datanucleus.fixedDatastore</name>
              <value>true</value>
            </property>

            <property>
             <name>datanucleus.autoCreateTables</name>
             <value>True</value>
             </property>

             <property>
                <name>javax.jdo.option.ConnectionPassword</name>
                <value>root</value>
                <description>password to use against metastore database</description>
              </property>


             <property>
                <name>javax.jdo.option.ConnectionUserName</name>
                <value>root</value>
                <description>Username to use against metastore database</description>
              </property>
          #############################